Merge branch 'master' into job-service

This commit is contained in:
Tan Jiang 2016-05-10 19:45:24 +08:00
commit 2752213ed1
34 changed files with 1726 additions and 631 deletions

1
.gitignore vendored
View File

@ -5,3 +5,4 @@ Deploy/config/ui/app.conf
Deploy/config/db/env
Deploy/harbor.cfg
ui/ui
*.pyc

View File

@ -1,14 +1,14 @@
language: go
go:
- 1.5.3
- 1.6.2
go_import_path: github.com/vmware/harbor
service:
- mysql
env: GO15VENDOREXPERIMENT=1 DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD=
env: DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD=
install:
- sudo apt-get update && sudo apt-get install -y libldap2-dev

View File

@ -158,3 +158,9 @@ create table properties (
insert into properties (k, v) values
('schema_version', '0.1.1');
CREATE TABLE IF NOT EXISTS `alembic_version` (
`version_num` varchar(32) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
insert into alembic_version values ('0.1.1');

View File

@ -116,7 +116,7 @@ FNULL = open(os.devnull, 'w')
from functools import wraps
def stat_decorator(func):
#@wraps(func)
@wraps(func)
def check_wrapper(*args, **kwargs):
stat = func(*args, **kwargs)
message = "Generated configuration file: %s" % kwargs['path'] \

View File

@ -12,4 +12,4 @@ LDAP_URL=$ldap_url
LDAP_BASE_DN=$ldap_basedn
SELF_REGISTRATION=$self_registration
LOG_LEVEL=debug
GODEBUG=netdns=cgo
GODEBUG=netdns=cgo

View File

@ -1,4 +1,4 @@
FROM golang:1.5.1
FROM golang:1.6.2
MAINTAINER jiangd@vmware.com
@ -11,7 +11,6 @@ COPY . /go/src/github.com/vmware/harbor
COPY ./vendor/golang.org /go/src/golang.org
WORKDIR /go/src/github.com/vmware/harbor/ui
ENV GO15VENDOREXPERIMENT 1
RUN go get -d github.com/docker/distribution \
&& go get -d github.com/docker/libtrust \
&& go get -d github.com/go-sql-driver/mysql \

View File

@ -23,6 +23,7 @@ import (
"strings"
"time"
"github.com/docker/distribution/manifest/schema1"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
svc_utils "github.com/vmware/harbor/service/utils"
@ -40,61 +41,20 @@ type RepositoryAPI struct {
BaseAPI
userID int
username string
registry *registry.Registry
}
// Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission.
func (ra *RepositoryAPI) Prepare() {
userID, ok := ra.GetSession("userId").(int)
if !ok {
ra.userID = dao.NonExistUserID
} else {
ra.userID = userID
userID = dao.NonExistUserID
}
ra.userID = userID
username, ok := ra.GetSession("username").(string)
if !ok {
log.Warning("failed to get username from session")
ra.username = ""
} else {
if ok {
ra.username = username
}
var client *http.Client
//no session, initialize a standard auth handler
if ra.userID == dao.NonExistUserID && len(ra.username) == 0 {
username, password, _ := ra.Ctx.Request.BasicAuth()
credential := auth.NewBasicAuthCredential(username, password)
client = registry.NewClientStandardAuthHandlerEmbeded(credential)
log.Debug("initializing standard auth handler")
} else {
// session works, initialize a username auth handler
username := ra.username
if len(username) == 0 {
user, err := dao.GetUser(models.User{
UserID: ra.userID,
})
if err != nil {
log.Errorf("error occurred whiling geting user for initializing a username auth handler: %v", err)
return
}
username = user.Username
}
client = registry.NewClientUsernameAuthHandlerEmbeded(username)
log.Debug("initializing username auth handler: %s", username)
}
endpoint := os.Getenv("REGISTRY_URL")
r, err := registry.New(endpoint, client)
if err != nil {
log.Fatalf("error occurred while initializing auth handler for repository API: %v", err)
}
ra.registry = r
}
// Get ...
@ -156,10 +116,16 @@ func (ra *RepositoryAPI) Delete() {
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
tags := []string{}
tag := ra.GetString("tag")
if len(tag) == 0 {
tagList, err := ra.registry.ListTag(repoName)
tagList, err := rc.ListTag()
if err != nil {
e, ok := errors.ParseError(err)
if ok {
@ -169,16 +135,14 @@ func (ra *RepositoryAPI) Delete() {
log.Error(err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
}
tags = append(tags, tagList...)
} else {
tags = append(tags, tag)
}
for _, t := range tags {
if err := ra.registry.DeleteTag(repoName, t); err != nil {
if err := rc.DeleteTag(t); err != nil {
e, ok := errors.ParseError(err)
if ok {
ra.CustomAbort(e.StatusCode, e.Message)
@ -206,15 +170,23 @@ type tag struct {
// GetTags handles GET /api/repositories/tags
func (ra *RepositoryAPI) GetTags() {
var tags []string
repoName := ra.GetString("repo_name")
tags, err := ra.registry.ListTag(repoName)
if len(repoName) == 0 {
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
tags := []string{}
ts, err := rc.ListTag()
if err != nil {
e, ok := errors.ParseError(err)
if ok {
log.Info(e)
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
@ -222,6 +194,8 @@ func (ra *RepositoryAPI) GetTags() {
}
}
tags = append(tags, ts...)
ra.Data["json"] = tags
ra.ServeJSON()
}
@ -231,13 +205,23 @@ func (ra *RepositoryAPI) GetManifests() {
repoName := ra.GetString("repo_name")
tag := ra.GetString("tag")
if len(repoName) == 0 || len(tag) == 0 {
ra.CustomAbort(http.StatusBadRequest, "repo_name or tag is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
item := models.RepoItem{}
_, _, payload, err := ra.registry.PullManifest(repoName, tag, registry.ManifestVersion1)
mediaTypes := []string{schema1.MediaTypeManifest}
_, _, payload, err := rc.PullManifest(tag, mediaTypes)
if err != nil {
e, ok := errors.ParseError(err)
if ok {
log.Info(e)
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
@ -264,3 +248,31 @@ func (ra *RepositoryAPI) GetManifests() {
ra.Data["json"] = item
ra.ServeJSON()
}
func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) {
endpoint := os.Getenv("REGISTRY_URL")
//no session, use basic auth
if ra.userID == dao.NonExistUserID {
username, password, _ := ra.Ctx.Request.BasicAuth()
credential := auth.NewBasicAuthCredential(username, password)
return registry.NewRepositoryWithCredential(repoName, endpoint, credential)
}
//session exists, use username
if len(ra.username) == 0 {
u := models.User{
UserID: ra.userID,
}
user, err := dao.GetUser(u)
if err != nil {
return nil, err
}
ra.username = user.Username
}
return registry.NewRepositoryWithUsername(repoName, endpoint, ra.username)
}

View File

@ -187,7 +187,9 @@ func (ua *UserAPI) Delete() {
// ChangePassword handles PUT to /api/users/{}/password
func (ua *UserAPI) ChangePassword() {
if !(ua.AuthMode == "db_auth") {
ldapAdminUser := (ua.AuthMode == "ldap_auth" && ua.userID == 1 && ua.userID == ua.currentUserID)
if !(ua.AuthMode == "db_auth" || ldapAdminUser) {
ua.CustomAbort(http.StatusForbidden, "")
}

View File

@ -41,6 +41,7 @@ type BaseController struct {
beego.Controller
i18n.Locale
SelfRegistration bool
IsLdapAdminUser bool
IsAdmin bool
AuthMode string
}
@ -115,7 +116,11 @@ func (b *BaseController) Prepare() {
if sessionUserID != nil {
b.Data["Username"] = b.GetSession("username")
b.Data["UserId"] = sessionUserID.(int)
if (sessionUserID == 1 && b.AuthMode == "ldap_auth") {
b.IsLdapAdminUser = true
}
var err error
b.IsAdmin, err = dao.IsAdminRole(sessionUserID.(int))
if err != nil {
@ -126,6 +131,7 @@ func (b *BaseController) Prepare() {
b.Data["IsAdmin"] = b.IsAdmin
b.Data["SelfRegistration"] = b.SelfRegistration
b.Data["IsLdapAdminUser"] = b.IsLdapAdminUser
}

View File

@ -62,6 +62,14 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) {
sql += ` and u.username like ? `
queryParam = append(queryParam, accessLog.Username)
}
if accessLog.RepoName != "" {
sql += ` and a.repo_name = ? `
queryParam = append(queryParam, accessLog.RepoName)
}
if accessLog.RepoTag != "" {
sql += ` and a.repo_tag = ? `
queryParam = append(queryParam, accessLog.RepoTag)
}
if accessLog.Keywords != "" {
sql += ` and a.operation in ( `
keywordList := strings.Split(accessLog.Keywords, "/")

View File

@ -100,6 +100,8 @@ func clearUp(username string) {
const username string = "Tester01"
const projectName string = "test_project"
const repoTag string = "test1.1"
const repoTag2 string = "test1.2"
const SysAdmin int = 1
const projectAdmin int = 2
const developer int = 3
@ -417,6 +419,66 @@ func TestGetAccessLog(t *testing.T) {
}
}
func TestAddAccessLog(t *testing.T) {
var err error
var accessLogList []models.AccessLog
accessLog := models.AccessLog{
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
RepoName: currentProject.Name + "/",
RepoTag: repoTag,
GUID: "N/A",
Operation: "create",
OpTime: time.Now(),
}
err = AddAccessLog(accessLog)
if err != nil {
t.Errorf("Error occurred in AddAccessLog: %v", err)
}
accessLogList, err = GetAccessLogs(accessLog)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
if len(accessLogList) != 1 {
t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList))
}
if accessLogList[0].RepoName != projectName+"/" {
t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName)
}
if accessLogList[0].RepoTag != repoTag {
t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag, accessLogList[0].RepoTag)
}
}
func TestAccessLog(t *testing.T) {
var err error
var accessLogList []models.AccessLog
accessLog := models.AccessLog{
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
RepoName: currentProject.Name + "/",
RepoTag: repoTag2,
Operation: "create",
}
err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/", repoTag2, "create")
if err != nil {
t.Errorf("Error occurred in AccessLog: %v", err)
}
accessLogList, err = GetAccessLogs(accessLog)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
if len(accessLogList) != 1 {
t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList))
}
if accessLogList[0].RepoName != projectName+"/" {
t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName)
}
if accessLogList[0].RepoTag != repoTag2 {
t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag2, accessLogList[0].RepoTag)
}
}
func TestProjectExists(t *testing.T) {
var exists bool
var err error

View File

@ -109,7 +109,7 @@ paths:
tags:
- Products
responses:
200:
201:
description: Project created successfully.
400:
description: Unsatisfied with constraints of the project creation.
@ -383,6 +383,28 @@ paths:
description: User does not have permission of admin role.
500:
description: Unexpected internal errors.
post:
summary: Creates a new user account.
description: |
This endpoint is to create a user if the user does not already exist.
parameters:
- name: user
in: body
description: New created user.
required: true
schema:
$ref: '#/definitions/User'
tags:
- Products
responses:
201:
description: User created successfully.
400:
description: Unsatisfied with constraints of the user creation.
403:
description: User registration can only be used by admin role user when self-registration is off.
500:
description: Unexpected internal errors.
/users/{user_id}:
put:
summary: Update a registered user to change to be an administrator of Harbor.
@ -438,6 +460,37 @@ paths:
description: User ID does not exist.
500:
description: Unexpected internal errors.
/users/{user_id}/password:
put:
summary: Change the password on a user that already exists.
description: |
This endpoint is for user to update password. Users with the admin role can change any user's password. Guest users can change only their own password.
parameters:
- name: user_id
in: path
type: integer
format: int32
required: true
description: Registered user ID.
- name: password
in: body
description: Password to be updated.
required: true
schema:
$ref: '#/definitions/Password'
tags:
- Products
responses:
200:
description: Updated password successfully.
400:
description: Invalid user ID; Old password is blank; New password is blank.
401:
description: Old password is not correct.
403:
description: Guests can only change their own account.
500:
description: Unexpected internal errors.
/repositories:
get:
summary: Get repositories accompany with relevant project and repo name.
@ -640,6 +693,15 @@ definitions:
deleted:
type: integer
format: int32
Password:
type: object
properties:
old_password:
type: string
description: The user's existing password.
new_password:
type: string
description: New password for marking as to be updated.
AccessLog:
type: object
properties:

23
migration/Dockerfile Normal file
View File

@ -0,0 +1,23 @@
FROM mysql:5.6
MAINTAINER bhe@vmware.com
RUN sed -i -e 's/us.archive.ubuntu.com/archive.ubuntu.com/g' /etc/apt/sources.list
RUN apt-get update
RUN apt-get install -y curl python python-pip git python-mysqldb
RUN pip install alembic
RUN mkdir -p /harbor-migration
WORKDIR /harbor-migration
COPY ./ ./
COPY ./migration.cfg ./
RUN ./prepare.sh
ENTRYPOINT ["./run.sh"]

51
migration/README.md Normal file
View File

@ -0,0 +1,51 @@
# migration
Migration is a module for migrating database schema between different version of project [harbor](https://github.com/vmware/harbor)
**WARNING!!** You must backup your data before migrating
###installation
- step 1: modify migration.cfg
- step 2: build image from dockerfile
```
cd harbor-migration
docker build -t your-image-name .
```
###migration operation
- show instruction of harbor-migration
```docker run your-image-name help```
- create backup file in `/path/to/backup`
```
docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name backup
```
- restore from backup file in `/path/to/backup`
```
docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name restore
```
- perform database schema upgrade
```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head```
- perform database schema downgrade(downgrade has been disabled)
```docker run -v /data/database:/var/lib/mysql your-image-name down base```
###migration step
- step 1: stop and remove harbor service
```
docker-compose stop && docker-compose rm -f
```
- step 2: perform migration operation
- step 3: rebuild newest harbor images and restart service
```
docker-compose build && docker-compose up -d
```

4
migration/alembic.sql Normal file
View File

@ -0,0 +1,4 @@
use `registry`;
CREATE TABLE IF NOT EXISTS `alembic_version` (
`version_num` varchar(32) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

68
migration/alembic.tpl Normal file
View File

@ -0,0 +1,68 @@
echo "
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = migration_harbor
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; this defaults
# to migration_harbor/versions. When using multiple version
# directories, initial revisions must be specified with --version-path
# version_locations = %(here)s/bar %(here)s/bat migration_harbor/versions
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
sqlalchemy.url = mysql://$db_username:$db_password@localhost:$db_port/$db_name
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S"

4
migration/migration.cfg Normal file
View File

@ -0,0 +1,4 @@
db_username="root"
db_password="root123"
db_port="3306"
db_name="registry"

View File

@ -0,0 +1,85 @@
# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,128 @@
# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""0.1.0 to 0.1.1
Revision ID: 0.1.1
Revises:
Create Date: 2016-04-18 18:32:14.101897
"""
# revision identifiers, used by Alembic.
revision = '0.1.1'
down_revision = None
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship
from datetime import datetime
Session = sessionmaker()
Base = declarative_base()
class Properties(Base):
__tablename__ = 'properties'
k = sa.Column(sa.String(64), primary_key = True)
v = sa.Column(sa.String(128), nullable = False)
class ProjectMember(Base):
__tablename__ = 'project_member'
project_id = sa.Column(sa.Integer(), primary_key = True)
user_id = sa.Column(sa.Integer(), primary_key = True)
role = sa.Column(sa.Integer(), nullable = False)
creation_time = sa.Column(sa.DateTime(), nullable = True)
update_time = sa.Column(sa.DateTime(), nullable = True)
sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ),
sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ),
sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ),
class UserProjectRole(Base):
__tablename__ = 'user_project_role'
upr_id = sa.Column(sa.Integer(), primary_key = True)
user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id'))
pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id'))
project_role = relationship("ProjectRole")
class ProjectRole(Base):
__tablename__ = 'project_role'
pr_id = sa.Column(sa.Integer(), primary_key = True)
project_id = sa.Column(sa.Integer(), nullable = False)
role_id = sa.Column(sa.Integer(), nullable = False)
sa.ForeignKeyConstraint(['role_id'], [u'role.role_id'])
sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'])
class Access(Base):
__tablename__ = 'access'
access_id = sa.Column(sa.Integer(), primary_key = True)
access_code = sa.Column(sa.String(1))
comment = sa.Column(sa.String(30))
def upgrade():
"""
update schema&data
"""
bind = op.get_bind()
session = Session(bind=bind)
#delete M from table access
acc = session.query(Access).filter_by(access_id=1).first()
session.delete(acc)
#create table property
Properties.__table__.create(bind)
session.add(Properties(k='schema_version', v='0.1.1'))
#create table project_member
ProjectMember.__table__.create(bind)
#fill data
join_result = session.query(UserProjectRole).join(UserProjectRole.project_role).all()
for result in join_result:
session.add(ProjectMember(project_id=result.project_role.project_id, \
user_id=result.user_id, role=result.project_role.role_id, \
creation_time=datetime.now(), update_time=datetime.now()))
#drop user_project_role table before drop project_role
#because foreign key constraint
op.drop_table('user_project_role')
op.drop_table('project_role')
#add column to table project
op.add_column('project', sa.Column('update_time', sa.DateTime(), nullable=True))
#add column to table role
op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False))
#add column to table user
op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True))
op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True))
op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True))
session.commit()
def downgrade():
"""
Downgrade has been disabled.
"""
pass

3
migration/prepare.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
source ./migration.cfg
source ./alembic.tpl > ./alembic.ini

94
migration/run.sh Executable file
View File

@ -0,0 +1,94 @@
#!/bin/bash
source ./migration.cfg
WAITTIME=60
DBCNF="-hlocalhost -u${db_username}"
#prevent shell to print insecure message
export MYSQL_PWD="${db_password}"
if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then
echo "Usage:"
echo "backup perform database backup"
echo "restore perform database restore"
echo "up, upgrade perform database schema upgrade"
echo "h, help usage help"
exit 0
fi
if [[ $1 = "up" || $1 = "upgrade" ]]; then
echo "Please backup before upgrade."
read -p "Enter y to continue updating or n to abort:" ans
case $ans in
[Yy]* )
;;
[Nn]* )
exit 0
;;
* ) echo "illegal answer: $ans. Upgrade abort!!"
exit 1
;;
esac
fi
echo 'Trying to start mysql server...'
DBRUN=0
nohup mysqld 2>&1 > ./nohup.log&
for i in $(seq 1 $WAITTIME); do
echo "$(/usr/sbin/service mysql status)"
if [[ "$(/usr/sbin/service mysql status)" =~ "not running" ]]; then
sleep 1
else
DBRUN=1
break
fi
done
if [[ $DBRUN -eq 0 ]]; then
echo "timeout. Can't run mysql server."
exit 1
fi
key="$1"
case $key in
up|upgrade)
VERSION="$2"
if [[ -z $VERSION ]]; then
VERSION="head"
echo "Version is not specified. Default version is head."
fi
echo "Performing upgrade ${VERSION}..."
if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \
where table_schema='registry' and table_name='alembic_version';") -eq 0 ]]; then
echo "table alembic_version does not exist. Trying to initial alembic_version."
mysql $DBCNF < ./alembic.sql
#compatible with version 0.1.0 and 0.1.1
if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \
where table_schema='registry' and table_name='properties'") -eq 0 ]]; then
echo "table properties does not exist. The version of registry is 0.1.0"
else
echo "The version of registry is 0.1.1"
mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')"
fi
fi
alembic -c ./alembic.ini upgrade ${VERSION}
echo "Upgrade performed."
;;
backup)
echo "Performing backup..."
mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql
echo "Backup performed."
;;
restore)
echo "Performing restore..."
mysql $DBCNF < ./backup/registry.sql
echo "Restore performed."
;;
*)
echo "unknown option"
exit 0
;;
esac

View File

@ -39,6 +39,7 @@ const (
// GetResourceActions ...
func GetResourceActions(scopes []string) []*token.ResourceActions {
log.Debugf("scopes: %+v", scopes)
var res []*token.ResourceActions
for _, s := range scopes {
if s == "" {
@ -59,6 +60,7 @@ func GetResourceActions(scopes []string) []*token.ResourceActions {
func FilterAccess(username string, authenticated bool, a *token.ResourceActions) {
if a.Type == "registry" && a.Name == "catalog" {
log.Infof("current access, type: %s, name:%s, actions:%v \n", a.Type, a.Name, a.Actions)
return
}
@ -108,7 +110,7 @@ func FilterAccess(username string, authenticated bool, a *token.ResourceActions)
}
// GenTokenForUI is for the UI process to call, so it won't establish a https connection from UI to proxy.
func GenTokenForUI(username string, service string, scopes []string) (string, error) {
func GenTokenForUI(username string, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
access := GetResourceActions(scopes)
for _, a := range access {
FilterAccess(username, true, a)
@ -117,22 +119,22 @@ func GenTokenForUI(username string, service string, scopes []string) (string, er
}
// MakeToken makes a valid jwt token based on parms.
func MakeToken(username, service string, access []*token.ResourceActions) (string, error) {
func MakeToken(username, service string, access []*token.ResourceActions) (token string, expiresIn int, issuedAt *time.Time, err error) {
pk, err := libtrust.LoadKeyFile(privateKey)
if err != nil {
return "", err
return "", 0, nil, err
}
tk, err := makeTokenCore(issuer, username, service, expiration, access, pk)
tk, expiresIn, issuedAt, err := makeTokenCore(issuer, username, service, expiration, access, pk)
if err != nil {
return "", err
return "", 0, nil, err
}
rs := fmt.Sprintf("%s.%s", tk.Raw, base64UrlEncode(tk.Signature))
return rs, nil
return rs, expiresIn, issuedAt, nil
}
//make token core
func makeTokenCore(issuer, subject, audience string, expiration int,
access []*token.ResourceActions, signingKey libtrust.PrivateKey) (*token.Token, error) {
access []*token.ResourceActions, signingKey libtrust.PrivateKey) (t *token.Token, expiresIn int, issuedAt *time.Time, err error) {
joseHeader := &token.Header{
Type: "JWT",
@ -142,10 +144,12 @@ func makeTokenCore(issuer, subject, audience string, expiration int,
jwtID, err := randString(16)
if err != nil {
return nil, fmt.Errorf("Error to generate jwt id: %s", err)
return nil, 0, nil, fmt.Errorf("Error to generate jwt id: %s", err)
}
now := time.Now()
now := time.Now().UTC()
issuedAt = &now
expiresIn = expiration * 60
claimSet := &token.ClaimSet{
Issuer: issuer,
@ -161,10 +165,10 @@ func makeTokenCore(issuer, subject, audience string, expiration int,
var joseHeaderBytes, claimSetBytes []byte
if joseHeaderBytes, err = json.Marshal(joseHeader); err != nil {
return nil, fmt.Errorf("unable to marshal jose header: %s", err)
return nil, 0, nil, fmt.Errorf("unable to marshal jose header: %s", err)
}
if claimSetBytes, err = json.Marshal(claimSet); err != nil {
return nil, fmt.Errorf("unable to marshal claim set: %s", err)
return nil, 0, nil, fmt.Errorf("unable to marshal claim set: %s", err)
}
encodedJoseHeader := base64UrlEncode(joseHeaderBytes)
@ -173,12 +177,13 @@ func makeTokenCore(issuer, subject, audience string, expiration int,
var signatureBytes []byte
if signatureBytes, _, err = signingKey.Sign(strings.NewReader(payload), crypto.SHA256); err != nil {
return nil, fmt.Errorf("unable to sign jwt payload: %s", err)
return nil, 0, nil, fmt.Errorf("unable to sign jwt payload: %s", err)
}
signature := base64UrlEncode(signatureBytes)
tokenString := fmt.Sprintf("%s.%s", payload, signature)
return token.NewToken(tokenString)
t, err = token.NewToken(tokenString)
return
}
func randString(length int) (string, error) {

View File

@ -17,6 +17,7 @@ package token
import (
"net/http"
"time"
"github.com/vmware/harbor/auth"
"github.com/vmware/harbor/models"
@ -43,7 +44,6 @@ func (h *Handler) Get() {
authenticated := authenticate(username, password)
service := h.GetString("service")
scopes := h.GetStrings("scope")
log.Debugf("scopes: %+v", scopes)
if len(scopes) == 0 && !authenticated {
log.Info("login request with invalid credentials")
@ -59,14 +59,16 @@ func (h *Handler) Get() {
func (h *Handler) serveToken(username, service string, access []*token.ResourceActions) {
writer := h.Ctx.ResponseWriter
//create token
rawToken, err := MakeToken(username, service, access)
rawToken, expiresIn, issuedAt, err := MakeToken(username, service, access)
if err != nil {
log.Errorf("Failed to make token, error: %v", err)
writer.WriteHeader(http.StatusInternalServerError)
return
}
tk := make(map[string]string)
tk := make(map[string]interface{})
tk["token"] = rawToken
tk["expires_in"] = expiresIn
tk["issued_at"] = issuedAt.Format(time.RFC3339)
h.Data["json"] = tk
h.ServeJSON()
}

View File

@ -25,10 +25,14 @@ import (
"github.com/astaxie/beego/cache"
)
// Cache is the global cache in system.
var Cache cache.Cache
var registryClient *registry.Registry
var (
// Cache is the global cache in system.
Cache cache.Cache
endpoint string
username string
registryClient *registry.Registry
repositoryClients map[string]*registry.Repository
)
const catalogKey string = "catalog"
@ -39,17 +43,25 @@ func init() {
log.Errorf("Failed to initialize cache, error:%v", err)
}
endpoint := os.Getenv("REGISTRY_URL")
client := registry.NewClientUsernameAuthHandlerEmbeded("admin")
registryClient, err = registry.New(endpoint, client)
if err != nil {
log.Fatalf("error occurred while initializing authentication handler used by cache: %v", err)
}
endpoint = os.Getenv("REGISTRY_URL")
username = "admin"
repositoryClients = make(map[string]*registry.Repository, 10)
}
// RefreshCatalogCache calls registry's API to get repository list and write it to cache.
func RefreshCatalogCache() error {
log.Debug("refreshing catalog cache...")
if registryClient == nil {
var err error
registryClient, err = registry.NewRegistryWithUsername(endpoint, username)
if err != nil {
log.Errorf("error occurred while initializing registry client used by cache: %v", err)
return err
}
}
var err error
rs, err := registryClient.Catalog()
if err != nil {
return err
@ -58,10 +70,19 @@ func RefreshCatalogCache() error {
repos := []string{}
for _, repo := range rs {
tags, err := registryClient.ListTag(repo)
rc, ok := repositoryClients[repo]
if !ok {
rc, err = registry.NewRepositoryWithUsername(repo, endpoint, username)
if err != nil {
log.Errorf("error occurred while initializing repository client used by cache: %s %v", repo, err)
continue
}
repositoryClients[repo] = rc
}
tags, err := rc.ListTag()
if err != nil {
log.Errorf("error occurred while list tag for %s: %v", repo, err)
return err
continue
}
if len(tags) != 0 {

View File

@ -0,0 +1,60 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"net/http"
au "github.com/docker/distribution/registry/client/auth"
)
// Handler authorizes requests according to the schema
type Handler interface {
// Scheme : basic, bearer
Scheme() string
//AuthorizeRequest adds basic auth or token auth to the header of request
AuthorizeRequest(req *http.Request, params map[string]string) error
}
// RequestAuthorizer holds a handler list, which will authorize request.
// Implements interface RequestModifier
type RequestAuthorizer struct {
handlers []Handler
challenges []au.Challenge
}
// NewRequestAuthorizer ...
func NewRequestAuthorizer(handlers []Handler, challenges []au.Challenge) *RequestAuthorizer {
return &RequestAuthorizer{
handlers: handlers,
challenges: challenges,
}
}
// ModifyRequest adds authorization to the request
func (r *RequestAuthorizer) ModifyRequest(req *http.Request) error {
for _, handler := range r.handlers {
for _, challenge := range r.challenges {
if handler.Scheme() == challenge.Scheme {
if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
return err
}
}
}
}
return nil
}

View File

@ -0,0 +1,44 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"net/http"
)
// Credential ...
type Credential interface {
// AddAuthorization adds authorization information to request
AddAuthorization(req *http.Request)
}
// Implements interface Credential
type basicAuthCredential struct {
username string
password string
}
// NewBasicAuthCredential ...
func NewBasicAuthCredential(username, password string) Credential {
return &basicAuthCredential{
username: username,
password: password,
}
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
}

View File

@ -1,197 +0,0 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
token_util "github.com/vmware/harbor/service/token"
"github.com/vmware/harbor/utils/log"
registry_errors "github.com/vmware/harbor/utils/registry/errors"
)
const (
// credential type
basicAuth string = "basic_auth"
secretKey string = "secret_key"
)
// Handler authorizes the request when encounters a 401 error
type Handler interface {
// Schema : basic, bearer
Schema() string
//AuthorizeRequest adds basic auth or token auth to the header of request
AuthorizeRequest(req *http.Request, params map[string]string) error
}
// Credential ...
type Credential interface {
// AddAuthorization adds authorization information to request
AddAuthorization(req *http.Request)
}
type basicAuthCredential struct {
username string
password string
}
// NewBasicAuthCredential ...
func NewBasicAuthCredential(username, password string) Credential {
return &basicAuthCredential{
username: username,
password: password,
}
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
}
type token struct {
Token string `json:"token"`
}
type standardTokenHandler struct {
client *http.Client
credential Credential
}
// NewStandardTokenHandler returns a standard token handler. The handler will request a token
// from token server whose URL is specified in the "WWW-authentication" header and add it to
// the origin request
// TODO deal with https
func NewStandardTokenHandler(credential Credential) Handler {
return &standardTokenHandler{
client: &http.Client{
Transport: http.DefaultTransport,
},
credential: credential,
}
}
// Schema implements the corresponding method in interface AuthHandler
func (t *standardTokenHandler) Schema() string {
return "bearer"
}
// AuthorizeRequest implements the corresponding method in interface AuthHandler
func (t *standardTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
realm, ok := params["realm"]
if !ok {
return errors.New("no realm")
}
service := params["service"]
scope := params["scope"]
u, err := url.Parse(realm)
if err != nil {
return err
}
q := u.Query()
q.Add("service", service)
for _, s := range strings.Split(scope, " ") {
q.Add("scope", s)
}
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return err
}
t.credential.AddAuthorization(r)
resp, err := t.client.Do(r)
if err != nil {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return registry_errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
decoder := json.NewDecoder(resp.Body)
tk := &token{}
if err = decoder.Decode(tk); err != nil {
return err
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", tk.Token))
log.Debugf("standardTokenHandler generated token successfully | %s %s", req.Method, req.URL)
return nil
}
type usernameTokenHandler struct {
username string
}
// NewUsernameTokenHandler returns a handler which will generate
// a token according the user's privileges
func NewUsernameTokenHandler(username string) Handler {
return &usernameTokenHandler{
username: username,
}
}
// Schema implements the corresponding method in interface AuthHandler
func (u *usernameTokenHandler) Schema() string {
return "bearer"
}
// AuthorizeRequest implements the corresponding method in interface AuthHandler
func (u *usernameTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
service := params["service"]
scopes := []string{}
scope := params["scope"]
if len(scope) != 0 {
scopes = strings.Split(scope, " ")
}
token, err := token_util.GenTokenForUI(u.username, service, scopes)
if err != nil {
return err
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token))
log.Debugf("usernameTokenHandler generated token successfully | %s %s", req.Method, req.URL)
return nil
}

View File

@ -0,0 +1,230 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
token_util "github.com/vmware/harbor/service/token"
"github.com/vmware/harbor/utils/log"
registry_errors "github.com/vmware/harbor/utils/registry/errors"
)
type scope struct {
Type string
Name string
Actions []string
}
func (s *scope) string() string {
return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ","))
}
type tokenGenerator func(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error)
// Implements interface Handler
type tokenHandler struct {
scope *scope
tg tokenGenerator
cache string // cached token
expiresIn int // The duration in seconds since the token was issued that it will remain valid
issuedAt *time.Time // The RFC3339-serialized UTC standard time at which a given token was issued
}
// Scheme returns the scheme that the handler can handle
func (t *tokenHandler) Scheme() string {
return "bearer"
}
// AuthorizeRequest will add authorization header which contains a token before the request is sent
func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
var scopes []*scope
var token string
hasFrom := false
from := req.URL.Query().Get("from")
if len(from) != 0 {
s := &scope{
Type: "repository",
Name: from,
Actions: []string{"pull"},
}
scopes = append(scopes, s)
// do not cache the token if "from" appears
hasFrom = true
}
scopes = append(scopes, t.scope)
expired := true
if t.expiresIn != 0 && t.issuedAt != nil {
expired = t.issuedAt.Add(time.Duration(t.expiresIn) * time.Second).Before(time.Now().UTC())
}
if expired || hasFrom {
scopeStrs := []string{}
for _, scope := range scopes {
scopeStrs = append(scopeStrs, scope.string())
}
to, expiresIn, issuedAt, err := t.tg(params["realm"], params["service"], scopeStrs)
if err != nil {
return err
}
token = to
if !hasFrom {
t.cache = token
t.expiresIn = expiresIn
t.issuedAt = issuedAt
log.Debug("add token to cache")
}
} else {
token = t.cache
log.Debug("get token from cache")
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token))
log.Debugf("add token to request: %s %s", req.Method, req.URL.String())
return nil
}
// Implements interface Handler
type standardTokenHandler struct {
tokenHandler
client *http.Client
credential Credential
}
// NewStandardTokenHandler returns a standard token handler. The handler will request a token
// from token server and add it to the origin request
// TODO deal with https
func NewStandardTokenHandler(credential Credential, scopeType, scopeName string, scopeActions ...string) Handler {
handler := &standardTokenHandler{
client: &http.Client{
Transport: http.DefaultTransport,
},
credential: credential,
}
handler.scope = &scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,
}
handler.tg = handler.generateToken
return handler
}
func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
u, err := url.Parse(realm)
if err != nil {
return
}
q := u.Query()
q.Add("service", service)
for _, scope := range scopes {
q.Add("scope", scope)
}
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return
}
s.credential.AddAuthorization(r)
resp, err := s.client.Do(r)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode != http.StatusOK {
err = registry_errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
tk := struct {
Token string `json:"token"`
ExpiresIn int `json:"expires_in"`
IssuedAt string `json:"issued_at"`
}{}
if err = json.Unmarshal(b, &tk); err != nil {
return
}
token = tk.Token
expiresIn = tk.ExpiresIn
t, err := time.Parse(time.RFC3339, tk.IssuedAt)
if err != nil {
log.Errorf("error occurred while parsing issued_at: %v", err)
err = nil
} else {
issuedAt = &t
}
log.Debug("get token from token server")
return
}
// Implements interface Handler
type usernameTokenHandler struct {
tokenHandler
username string
}
// NewUsernameTokenHandler returns a handler which will generate a token according to
// the user's privileges
func NewUsernameTokenHandler(username string, scopeType, scopeName string, scopeActions ...string) Handler {
handler := &usernameTokenHandler{
username: username,
}
handler.scope = &scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,
}
handler.tg = handler.generateToken
return handler
}
func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
token, expiresIn, issuedAt, err = token_util.GenTokenForUI(u.username, service, scopes)
log.Debug("get token by calling GenTokenForUI directly")
return
}

View File

@ -1,116 +0,0 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"net/http"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry/auth"
)
// NewClient returns a http.Client according to the handlers provided
func NewClient(handlers []auth.Handler) *http.Client {
transport := NewAuthTransport(http.DefaultTransport, handlers)
return &http.Client{
Transport: transport,
}
}
// NewClientStandardAuthHandlerEmbeded return a http.Client which will authorize the request
// according to the credential provided and send it again when encounters a 401 error
func NewClientStandardAuthHandlerEmbeded(credential auth.Credential) *http.Client {
handlers := []auth.Handler{}
tokenHandler := auth.NewStandardTokenHandler(credential)
handlers = append(handlers, tokenHandler)
return NewClient(handlers)
}
// NewClientUsernameAuthHandlerEmbeded return a http.Client which will authorize the request
// according to the user's privileges and send it again when encounters a 401 error
func NewClientUsernameAuthHandlerEmbeded(username string) *http.Client {
handlers := []auth.Handler{}
tokenHandler := auth.NewUsernameTokenHandler(username)
handlers = append(handlers, tokenHandler)
return NewClient(handlers)
}
type authTransport struct {
transport http.RoundTripper
handlers []auth.Handler
}
// NewAuthTransport wraps the AuthHandlers to be http.RounTripper
func NewAuthTransport(transport http.RoundTripper, handlers []auth.Handler) http.RoundTripper {
return &authTransport{
transport: transport,
handlers: handlers,
}
}
// RoundTrip ...
func (a *authTransport) RoundTrip(req *http.Request) (*http.Response, error) {
originResp, originErr := a.transport.RoundTrip(req)
if originErr != nil {
return originResp, originErr
}
log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL)
if originResp.StatusCode != http.StatusUnauthorized {
return originResp, nil
}
challenges := auth.ParseChallengeFromResponse(originResp)
reqChanged := false
for _, challenge := range challenges {
scheme := challenge.Scheme
for _, handler := range a.handlers {
if scheme != handler.Schema() {
log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema())
continue
}
if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
return nil, err
}
reqChanged = true
}
}
if !reqChanged {
log.Warning("no handler match scheme")
return originResp, nil
}
resp, err := a.transport.RoundTrip(req)
if err == nil {
log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL)
}
return resp, err
}

View File

@ -21,57 +21,68 @@ import (
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry/auth"
"github.com/vmware/harbor/utils/registry/errors"
)
// Registry holds information of a registry entiry
// Registry holds information of a registry entity
type Registry struct {
Endpoint *url.URL
client *http.Client
ub *uRLBuilder
}
type uRLBuilder struct {
root *url.URL
}
// NewRegistry returns an instance of registry
func NewRegistry(endpoint string, client *http.Client) (*Registry, error) {
endpoint = strings.TrimRight(endpoint, "/")
var (
// ManifestVersion1 : schema 1
ManifestVersion1 = manifest.Versioned{
SchemaVersion: 1,
MediaType: schema1.MediaTypeManifest,
}
// ManifestVersion2 : schema 2
ManifestVersion2 = manifest.Versioned{
SchemaVersion: 2,
MediaType: schema2.MediaTypeManifest,
}
)
// New returns an instance of Registry
func New(endpoint string, client *http.Client) (*Registry, error) {
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
return &Registry{
registry := &Registry{
Endpoint: u,
client: client,
ub: &uRLBuilder{
root: u,
},
}, nil
}
log.Debugf("initialized a registry client: %s", endpoint)
return registry, nil
}
// NewRegistryWithUsername returns a Registry instance which will authorize the request
// according to the privileges of user
func NewRegistryWithUsername(endpoint, username string) (*Registry, error) {
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
client, err := newClient(endpoint, username, nil, "registry", "catalog", "*")
if err != nil {
return nil, err
}
registry := &Registry{
Endpoint: u,
client: client,
}
log.Debugf("initialized a registry client with username: %s %s", endpoint, username)
return registry, nil
}
// Catalog ...
func (r *Registry) Catalog() ([]string, error) {
repos := []string{}
req, err := http.NewRequest("GET", r.ub.buildCatalogURL(), nil)
req, err := http.NewRequest("GET", buildCatalogURL(r.Endpoint.String()), nil)
if err != nil {
return repos, err
}
@ -108,209 +119,34 @@ func (r *Registry) Catalog() ([]string, error) {
}
}
// ListTag ...
func (r *Registry) ListTag(name string) ([]string, error) {
tags := []string{}
req, err := http.NewRequest("GET", r.ub.buildTagListURL(name), nil)
if err != nil {
return tags, err
}
resp, err := r.client.Do(req)
if err != nil {
return tags, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
if resp.StatusCode == http.StatusOK {
tagsResp := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResp); err != nil {
return tags, err
}
tags = tagsResp.Tags
return tags, nil
}
return tags, errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
func buildCatalogURL(endpoint string) string {
return fmt.Sprintf("%s/v2/_catalog", endpoint)
}
// ManifestExist ...
func (r *Registry) ManifestExist(name, reference string) (digest string, exist bool, err error) {
req, err := http.NewRequest("HEAD", r.ub.buildManifestURL(name, reference), nil)
func newClient(endpoint, username string, credential auth.Credential,
scopeType, scopeName string, scopeActions ...string) (*http.Client, error) {
endpoint = strings.TrimRight(endpoint, "/")
resp, err := http.Get(buildPingURL(endpoint))
if err != nil {
return
return nil, err
}
// request Schema 2 manifest, if the registry does not support it,
// Schema 1 manifest will be returned
req.Header.Set(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest)
resp, err := r.client.Do(req)
if err != nil {
return
var handlers []auth.Handler
var handler auth.Handler
if credential != nil {
handler = auth.NewStandardTokenHandler(credential, scopeType, scopeName, scopeActions...)
} else {
handler = auth.NewUsernameTokenHandler(username, scopeType, scopeName, scopeActions...)
}
if resp.StatusCode == http.StatusOK {
exist = true
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
handlers = append(handlers, handler)
if resp.StatusCode == http.StatusNotFound {
return
}
challenges := auth.ParseChallengeFromResponse(resp)
authorizer := auth.NewRequestAuthorizer(handlers, challenges)
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// PullManifest ...
func (r *Registry) PullManifest(name, reference string, version manifest.Versioned) (digest, mediaType string, payload []byte, err error) {
req, err := http.NewRequest("GET", r.ub.buildManifestURL(name, reference), nil)
if err != nil {
return
}
// if the registry does not support schema 2, schema 1 manifest will be returned
req.Header.Set(http.CanonicalHeaderKey("Accept"), version.MediaType)
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
payload = b
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// DeleteManifest ...
func (r *Registry) DeleteManifest(name, digest string) error {
req, err := http.NewRequest("DELETE", r.ub.buildManifestURL(name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// DeleteTag ...
func (r *Registry) DeleteTag(name, tag string) error {
digest, exist, err := r.ManifestExist(name, tag)
if err != nil {
return err
}
if !exist {
return errors.Error{
StatusCode: http.StatusNotFound,
}
}
return r.DeleteManifest(name, digest)
}
// DeleteBlob ...
func (r *Registry) DeleteBlob(name, digest string) error {
req, err := http.NewRequest("DELETE", r.ub.buildBlobURL(name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
func (u *uRLBuilder) buildCatalogURL() string {
return fmt.Sprintf("%s/v2/_catalog", u.root.String())
}
func (u *uRLBuilder) buildTagListURL(name string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", u.root.String(), name)
}
func (u *uRLBuilder) buildManifestURL(name, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", u.root.String(), name, reference)
}
func (u *uRLBuilder) buildBlobURL(name, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", u.root.String(), name, reference)
transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer})
return &http.Client{
Transport: transport,
}, nil
}

View File

@ -0,0 +1,505 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry/auth"
"github.com/vmware/harbor/utils/registry/errors"
)
// Repository holds information of a repository entity
type Repository struct {
Name string
Endpoint *url.URL
client *http.Client
}
// TODO add agent to header of request, notifications need it
// NewRepository returns an instance of Repository
func NewRepository(name, endpoint string, client *http.Client) (*Repository, error) {
name = strings.TrimSpace(name)
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
repository := &Repository{
Name: name,
Endpoint: u,
client: client,
}
return repository, nil
}
// NewRepositoryWithCredential returns a Repository instance which will authorize the request
// according to the credenttial
func NewRepositoryWithCredential(name, endpoint string, credential auth.Credential) (*Repository, error) {
name = strings.TrimSpace(name)
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
client, err := newClient(endpoint, "", credential, "repository", name, "pull", "push")
repository := &Repository{
Name: name,
Endpoint: u,
client: client,
}
log.Debugf("initialized a repository client with credential: %s %s", endpoint, name)
return repository, nil
}
// NewRepositoryWithUsername returns a Repository instance which will authorize the request
// according to the privileges of user
func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, error) {
name = strings.TrimSpace(name)
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
client, err := newClient(endpoint, username, nil, "repository", name, "pull", "push")
repository := &Repository{
Name: name,
Endpoint: u,
client: client,
}
log.Debugf("initialized a repository client with username: %s %s %s", endpoint, name, username)
return repository, nil
}
// ListTag ...
func (r *Repository) ListTag() ([]string, error) {
tags := []string{}
req, err := http.NewRequest("GET", buildTagListURL(r.Endpoint.String(), r.Name), nil)
if err != nil {
return tags, err
}
resp, err := r.client.Do(req)
if err != nil {
return tags, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
if resp.StatusCode == http.StatusOK {
tagsResp := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResp); err != nil {
return tags, err
}
tags = tagsResp.Tags
return tags, nil
}
return tags, errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// ManifestExist ...
func (r *Repository) ManifestExist(reference string) (digest string, exist bool, err error) {
req, err := http.NewRequest("HEAD", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema1.MediaTypeManifest)
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest)
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
exist = true
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
if resp.StatusCode == http.StatusNotFound {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// PullManifest ...
func (r *Repository) PullManifest(reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) {
req, err := http.NewRequest("GET", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
for _, mediaType := range acceptMediaTypes {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
}
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
payload = b
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// PushManifest ...
func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (digest string, err error) {
req, err := http.NewRequest("PUT", buildManifestURL(r.Endpoint.String(), r.Name, reference),
bytes.NewReader(payload))
if err != nil {
return
}
req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType)
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusCreated {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// DeleteManifest ...
func (r *Repository) DeleteManifest(digest string) error {
req, err := http.NewRequest("DELETE", buildManifestURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// DeleteTag ...
func (r *Repository) DeleteTag(tag string) error {
digest, exist, err := r.ManifestExist(tag)
if err != nil {
return err
}
if !exist {
return errors.Error{
StatusCode: http.StatusNotFound,
}
}
return r.DeleteManifest(digest)
}
// BlobExist ...
func (r *Repository) BlobExist(digest string) (bool, error) {
req, err := http.NewRequest("HEAD", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return false, err
}
resp, err := r.client.Do(req)
if err != nil {
return false, err
}
if resp.StatusCode == http.StatusOK {
return true, nil
}
if resp.StatusCode == http.StatusNotFound {
return false, nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, err
}
return false, errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// PullBlob ...
func (r *Repository) PullBlob(digest string) (size int64, data []byte, err error) {
req, err := http.NewRequest("GET", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return
}
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
size, err = strconv.ParseInt(contengLength, 10, 64)
if err != nil {
return
}
data = b
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) initiateBlobUpload(name string) (location, uploadUUID string, err error) {
req, err := http.NewRequest("POST", buildInitiateBlobUploadURL(r.Endpoint.String(), r.Name), nil)
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusAccepted {
location = resp.Header.Get(http.CanonicalHeaderKey("Location"))
uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID"))
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) monolithicBlobUpload(location, digest string, size int64, data []byte) error {
req, err := http.NewRequest("PUT", buildMonolithicBlobUploadURL(location, digest), bytes.NewReader(data))
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusCreated {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// PushBlob ...
func (r *Repository) PushBlob(digest string, size int64, data []byte) error {
exist, err := r.BlobExist(digest)
if err != nil {
return err
}
if exist {
log.Infof("blob already exists, skip pushing: %s %s", r.Name, digest)
return nil
}
location, _, err := r.initiateBlobUpload(r.Name)
if err != nil {
return err
}
return r.monolithicBlobUpload(location, digest, size, data)
}
// DeleteBlob ...
func (r *Repository) DeleteBlob(digest string) error {
req, err := http.NewRequest("DELETE", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
func buildPingURL(endpoint string) string {
return fmt.Sprintf("%s/v2/", endpoint)
}
func buildTagListURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repoName)
}
func buildManifestURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repoName, reference)
}
func buildBlobURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repoName, reference)
}
func buildInitiateBlobUploadURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repoName)
}
func buildMonolithicBlobUploadURL(location, digest string) string {
return fmt.Sprintf("%s&digest=%s", location, digest)
}

View File

@ -0,0 +1,59 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"net/http"
"github.com/vmware/harbor/utils/log"
)
// RequestModifier modifies request
type RequestModifier interface {
ModifyRequest(*http.Request) error
}
// Transport holds information about base transport and modifiers
type Transport struct {
transport http.RoundTripper
modifiers []RequestModifier
}
// NewTransport ...
func NewTransport(transport http.RoundTripper, modifiers []RequestModifier) *Transport {
return &Transport{
transport: transport,
modifiers: modifiers,
}
}
// RoundTrip ...
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
for _, modifier := range t.modifiers {
if err := modifier.ModifyRequest(req); err != nil {
return nil, err
}
}
resp, err := t.transport.RoundTrip(req)
if err != nil {
return nil, err
}
log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL.String())
return resp, err
}

View File

@ -57,6 +57,10 @@
<li><a id="aChangePassword" href="/changePassword" target="_blank"><span class="glyphicon glyphicon-pencil"></span>&nbsp;&nbsp;{{i18n .Lang "change_password"}}</a></li>
<li role="separator" class="divider"></li>
{{ end }}
{{ if eq .IsLdapAdminUser true }}
<li><a id="aChangePassword" href="/changePassword" target="_blank"><span class="glyphicon glyphicon-pencil"></span>&nbsp;&nbsp;{{i18n .Lang "change_password"}}</a></li>
<li role="separator" class="divider"></li>
{{ end }}
{{ if eq .AuthMode "db_auth" }}
{{ if eq .IsAdmin true }}
<li><a id="aAddUser" href="/addUser" target="_blank"><span class="glyphicon glyphicon-plus"></span>&nbsp;&nbsp;{{i18n .Lang "add_user"}}</a></li>