Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-22 00:27:44 +01:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source
This commit is contained in:
kunw 2016-07-14 14:35:37 +08:00
commit c1484cd0fa
8 changed files with 149 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@ -9,7 +9,8 @@
Project Harbor is an enterprise-class registry server, which extends the open source Docker Registry server by adding the functionality usually required by an enterprise, such as security, control, and management. Harbor is primarily designed to be a private registry - providing the needed security and control that enterprises require. It also helps minimize bandwidth usage, which is helpful to both improve productivity (local network access) as well as performance (for those with poor internet connectivity).
### Features
* **Role Based Access Control**: Users and Docker repositories are organized via "projects", a user can have different permission for images under a project.
* **Role based access control**: Users and Docker repositories are organized via "projects", a user can have different permission for images under a project.
* **Image replication**: Images can be replicated(synchronized) between multiple registry instances. Great for load balancing and distributed data centers.
* **Graphical user portal**: User can easily browse, search Docker repositories, manage projects/namespaces.
* **AD/LDAP support**: Harbor integrates with existing enterprise AD/LDAP for user authentication and management.
* **Auditing**: All the operations to the repositories are tracked.
@ -40,7 +41,7 @@ Harbor only works with docker 1.10+ and docker-compose 1.6.0+, and an internet-c
Generated configuration file: ./config/registry/config.yml
Generated configuration file: ./config/db/env
$ docker-compose up
$ docker-compose up -d
```
_If everything worked properly, you should be able to open a browser to visit the admin portal at http://reg.yourdomain.com . Note that the default administrator username/password are admin/Harbor12345 ._
@ -53,7 +54,7 @@ $ docker push reg.yourdomain.com/myproject/myrepo
**Data migration:**
The database schema can be updated in new version, so if you are upgrading from an older version with existing data, you need to migrate the existing data to new schema. For more details please refer to [Data Migration Guide](migration/README.md)
If you are upgrading Harbor from an older version with existing data, you may need to migrate the data to fit the new database schema. For more details please refer to [Data Migration Guide](docs/migration_guide.md) .
**NOTE:**
For those who don't want to clone the source, or need to install Harbor on a server not connected to the Internet - there is a pre-built installation package available. For details on how to download and use this installation package, please refer to [Installation and Configuration Guide](docs/installation_guide.md) .

14
api/target.go
View File

@ -190,14 +190,14 @@ func (t *TargetAPI) Post() {
t.CustomAbort(http.StatusConflict, "name is already used")
}
ta, err = dao.GetRepTargetByConnInfo(target.URL, target.Username)
ta, err = dao.GetRepTargetByEndpoint(target.URL)
if err != nil {
log.Errorf("failed to get target [ %s %s ]: %v", target.URL, target.Username, err)
log.Errorf("failed to get target [ %s ]: %v", target.URL, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if ta != nil {
t.CustomAbort(http.StatusConflict, "the connection information[ endpoint, username ] is conflict with other target")
t.CustomAbort(http.StatusConflict, fmt.Sprintf("the target whose endpoint is %s already exists", target.URL))
}
if len(target.Password) != 0 {
@ -260,15 +260,15 @@ func (t *TargetAPI) Put() {
}
}
if target.URL != originalTarget.URL || target.Username != originalTarget.Username {
ta, err := dao.GetRepTargetByConnInfo(target.URL, target.Username)
if target.URL != originalTarget.URL {
ta, err := dao.GetRepTargetByEndpoint(target.URL)
if err != nil {
log.Errorf("failed to get target [ %s %s ]: %v", target.URL, target.Username, err)
log.Errorf("failed to get target [ %s ]: %v", target.URL, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if ta != nil {
t.CustomAbort(http.StatusConflict, "the connection information[ endpoint, username ] is conflict with other target")
t.CustomAbort(http.StatusConflict, fmt.Sprintf("the target whose endpoint is %s already exists", target.URL))
}
}

3
api/utils.go
View File

@ -162,6 +162,9 @@ func TriggerReplicationByRepository(repository string, tags []string, operation
}
for _, policy := range policies {
if policy.Enabled == 0 {
continue
}
if err := TriggerReplication(policy.ID, repository, tags, operation); err != nil {
log.Errorf("failed to trigger replication of policy %d for %s: %v", policy.ID, repository, err)
} else {

16
dao/dao_test.go
View File

@ -855,6 +855,22 @@ func TestGetRepTargetByName(t *testing.T) {
}
}
func TestGetRepTargetByEndpoint(t *testing.T) {
target, err := GetRepTarget(targetID)
if err != nil {
t.Fatalf("failed to get target %d: %v", targetID, err)
}
target2, err := GetRepTargetByEndpoint(target.URL)
if err != nil {
t.Fatalf("failed to get target %s: %v", target.URL, err)
}
if target.URL != target2.URL {
t.Errorf("unexpected target URL: %s, expected: %s", target2.URL, target.URL)
}
}
func TestUpdateRepTarget(t *testing.T) {
target := &models.RepTarget{
Name: "name",

9
dao/replication_job.go
View File

@ -53,14 +53,13 @@ func GetRepTargetByName(name string) (*models.RepTarget, error) {
return &t, err
}
// GetRepTargetByConnInfo ...
func GetRepTargetByConnInfo(endpoint, username string) (*models.RepTarget, error) {
// GetRepTargetByEndpoint ...
func GetRepTargetByEndpoint(endpoint string) (*models.RepTarget, error) {
o := GetOrmer()
t := models.RepTarget{
URL: endpoint,
Username: username,
URL: endpoint,
}
err := o.Read(&t, "URL", "Username")
err := o.Read(&t, "URL")
if err == orm.ErrNoRows {
return nil, nil
}

103
docs/migration_guide.md Normal file
View File

@ -0,0 +1,103 @@
# Harbor upgrade and database migration guide
When upgrading your existing Habor instance to a newer version, you may need to migrate the data in your database. Refer to [change log](changelog.md) to find out whether there is any change in the database. If there is, you should go through the database migration process. Since the migration may alter the database schema, you should **always** back up your data before any migration.
*If your install Harbor for the first time, or the database version is the same as that of the lastest version, you do not need any database migration.*
**NOTE:** You must backup your data before any data migration.
### Upgrading Harbor and migrating data
1. Log in to the machine that Harbor runs on, stop and remove existing Harbor service if it is still running:
```
cd Deploy/
docker-compose down
```
2. Back up Harbor's current source code so that you can roll back to the current version when it is necessary.
```sh
cd ../..
mv harbor /tmp/harbor
```
3. Get the lastest source code from Github:
```sh
git clone https://github.com/vmware/harbor
```
4. Before upgrading Harbor, perform database migration first.
The directory **migration/** contains the tool for migration. The first step is to update values of `db_username`, `db_password`, `db_port`, `db_name` in **migration.cfg** so that they match your system's configuration.
5. The migration tool is delivered as a container, so you should build the image from its Dockerfile:
```
cd migration/
docker build -t migrate-tool .
```
6. Back up database to a directory such as `/path/to/backup`. You need to create the directory if it does not exist.
```
docker run -ti --rm -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup migrate-tool backup
```
7. Upgrade database schema and migrate data:
```
docker run -ti --rm -v /data/database:/var/lib/mysql migrate-tool up head
```
8. Change to `Deploy/` directory, configure Harbor by modifying the file `harbor.cfg`, you may need to refer to the configuration files you've backed up during step 2. Refer to [Installation & Configuration Guide ](../docs/installation_guide.md) for more info.
9. If HTTPS has been enabled for Harbor before, restore the `nginx.conf` and key/certificate files from the backup files in Step 2. Refer to [Configuring Harbor with HTTPS Access](../docs/configure_https.md) for more info.
10. Under the directory `Deploy/`, run the `./prepare` script to generate necessary config files.
11. Rebuild Harbor and restart the registry service
```
docker-compose up --build -d
```
### Roll back from an upgrade
For any reason, if you want to roll back to the previous version of Harbor, follow the below steps:
1. Stop and remove the current Harbor service if it is still running.
```
cd Deploy/
docker-compose down
```
2. Restore database from backup file in `/path/to/backup` .
```
docker run -ti --rm -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup migrate-tool restore
```
3. Remove current source code of Harbor.
```
rm -rf harbor
```
4. Restore the source code of an older version of Harbor.
```sh
mv /tmp/harbor harbor
```
5. Restart Harbor service using the previous configuration.
```sh
cd Deploy/
docker-compose up --build -d
```
### Migration tool reference
- Use `help` command to show instructions of the migration tool:
```docker run --rm migrate-tool help```
- Use `test` command to test mysql connection:
```docker run --rm -v /data/database:/var/lib/mysql migrate-tool test```

17
job/replication/transfer.go
View File

@ -28,6 +28,7 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry"
@ -192,7 +193,13 @@ enter:
return "", err
}
if !exist {
err := c.createProject()
project, err := dao.GetProjectByName(c.project)
if err != nil {
c.logger.Errorf("an error occurred while getting project %s on %s: %v", c.project, c.srcURL, err)
return "", err
}
err = c.createProject(project.Public == 1)
if err != nil {
// other job may be also doing the same thing when the current job
// is creating project, so when the response code is 409, re-check
@ -286,13 +293,13 @@ func (c *Checker) projectExist() (exist, canWrite bool, err error) {
return
}
func (c *Checker) createProject() error {
// TODO handle publicity of project
func (c *Checker) createProject(isPublic bool) error {
project := struct {
ProjectName string `json:"project_name"`
Public bool `json:"public"`
}{
ProjectName: c.project,
Public: isPublic,
}
data, err := json.Marshal(project)
@ -495,8 +502,8 @@ func (m *ManifestPusher) enter() (string, error) {
} else {
m.logger.Infof("manifest of %s:%s exists on source registry %s, continue manifest pushing", name, tag, m.srcURL)
_, manifestExist, err := m.dstClient.ManifestExist(m.digest)
if manifestExist {
digest, manifestExist, err := m.dstClient.ManifestExist(tag)
if manifestExist && digest == m.digest {
m.logger.Infof("manifest of %s:%s exists on destination registry %s, skip manifest pushing", name, tag, m.dstURL)
m.tags = m.tags[1:]

60
migration/README.md
View File

@ -1,60 +0,0 @@
# Migration guide
Migration is a module for migrating database schema between different version of project [Harbor](https://github.com/vmware/harbor)
This module is for those machine running Harbor's old version, such as 0.1.0. If your Harbor' version is up to date, please ignore this module.
**WARNING!!** You must backup your data before migrating
###Installation
- step 1:
```
cd migration
```
- step 2: change `db_username`, `db_password`, `db_port`, `db_name` in migration.cfg
- step 3: build image from dockerfile
```
docker build -t migrate-tool .
```
###Migrate Step
- step 1: stop and remove Harbor service
```
docker-compose down
```
- step 2: create backup file in `/path/to/backup`
```
docker run -ti --rm -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup migrate-tool backup
```
- step 3: perform database schema upgrade
```docker run -ti --rm -v /data/database:/var/lib/mysql migrate-tool up head```
- step 4: rebuild newest Harbor images and restart service
```
docker-compose build && docker-compose up -d
```
You may change `/data/database` to the mysql volumes path you set in docker-compose.yml.
###Migration operation reference
- You can use `help` to show instruction of Harbor migration
```docker run migrate-tool help```
- You can use `test` to test mysql connection in Harbor migration
```docker run --rm -v /data/database:/var/lib/mysql migrate-tool test```
- You can restore from backup file in `/path/to/backup`
```
docker run -ti --rm -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup migrate-tool restore
```