From 0461ad35066946b5346893ca798fb4df5d5f0c03 Mon Sep 17 00:00:00 2001 From: Henry Zhang Date: Tue, 12 Jul 2016 17:58:15 +0800 Subject: [PATCH 1/8] update migration guide --- migration/README.md | 80 +++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/migration/README.md b/migration/README.md index 454642c05..923ae82a0 100644 --- a/migration/README.md +++ b/migration/README.md @@ -1,59 +1,77 @@ -# Migration guide -Migration is a module for migrating database schema between different version of project [Harbor](https://github.com/vmware/harbor) +# Harbor upgrade and database migration guide -This module is for those machine running Harbor's old version, such as 0.1.0. If your Harbor' version is up to date, please ignore this module. +When upgrading your existing Habor instance to a newer version, you may need to migrate the data in your database. Refer to [change log](changelog.md) to find out whether there is any change in the database. If there is, you should go through the database migration process. Since the migration may alter the database schema, you should **always** back up your data before any migration. -**WARNING!!** You must backup your data before migrating +*If your install Harbor for the first time, or the database version is the same as that of the lastest version, you do not need any database migration.* -###Installation -- step 1: +**NOTE:** You must backup your data before any data migration. + +### Upgrading Harbor and migrating data + +1. Log in to the machine that Harbor runs on, back up Harbor's configuration files. + ```sh + mkdir -p /tmp/harbor/config + cp -r Deploy/config /tmp/harbor/config + cp Deploy/harbor.cfg /tmp/harbor ``` - cd migration - ``` -- step 2: change `db_username`, `db_password`, `db_port`, `db_name` in migration.cfg -- step 3: build image from dockerfile +2. Next, stop existing Harbor service if it is still running: + + ``` + cd ../../Deploy/ + docker-compose down ``` + +3. Get the lastest source code from Github: + ```sh + $ git clone https://github.com/vmware/harbor + ``` + +4. Before upgrading Harbor, perform data migration first. +The directory **migration/** contains the tool for migration. The first step is to update values of `db_username`, `db_password`, `db_port`, `db_name` in **migration.cfg** so that they match your system's configuration. + +5. The migration tool is delivered as a container, so you should build the image from its Dockerfile: + ``` + cd migration/harbor-migration + docker build -t migrate-tool . ``` -###Migrate Step -- step 1: stop and remove Harbor service - - ``` - docker-compose down - ``` -- step 2: create backup file in `/path/to/backup` +6. Back up database to a directory such as `/path/to/backup`. You need to create the directory if it does not exist. ``` docker run -ti --rm -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup migrate-tool backup ``` -- step 3: perform database schema upgrade - - ```docker run -ti --rm -v /data/database:/var/lib/mysql migrate-tool up head``` - - - -- step 4: rebuild newest Harbor images and restart service +7. Upgrade database schema and migrate data: ``` - docker-compose build && docker-compose up -d + docker run -ti --rm -v /data/database:/var/lib/mysql migrate-tool up head ``` -You may change `/data/database` to the mysql volumes path you set in docker-compose.yml. +8. Change to `Deploy/` directory, configure Harbor by updating the file `harbor.cfg`, you may need to refer to the configuration files you backed up during step 1. Refer to [Installation & Configuration Guide ](../docs/installation_guide.md) for more info. -###Migration operation reference -- You can use `help` to show instruction of Harbor migration +9. If HTTPS has been enabled for Harbor before, restore the `nginx.conf` and key/certificate files from the backup files in Step 1. Refer to [Configuring Harbor with HTTPS Access](../docs/configure_https.md) for more info. - ```docker run migrate-tool help``` +10. Run the `./prepare` script to generate necessary config files. + +11. Rebuild Harbor and restart the registry service + + ``` + docker-compose up --build -d + ``` + +### Migration tool reference +- Use `help` command to show instructions of the migration tool: + + ```docker run --rm migrate-tool help``` -- You can use `test` to test mysql connection in Harbor migration +- Use `test` command to test mysql connection: ```docker run --rm -v /data/database:/var/lib/mysql migrate-tool test``` -- You can restore from backup file in `/path/to/backup` +- Restore database from backup file in `/path/to/backup` ``` docker run -ti --rm -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup migrate-tool restore From 65676d089ec3e2d5f80e023af3c6741578846dc9 Mon Sep 17 00:00:00 2001 From: Henry Zhang Date: Tue, 12 Jul 2016 18:22:22 +0800 Subject: [PATCH 2/8] update migration guide --- migration/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/migration/README.md b/migration/README.md index 923ae82a0..fa400b3e3 100644 --- a/migration/README.md +++ b/migration/README.md @@ -16,24 +16,25 @@ When upgrading your existing Habor instance to a newer version, you may need to cp Deploy/harbor.cfg /tmp/harbor ``` -2. Next, stop existing Harbor service if it is still running: +2. Next, stop and remove existing Harbor service if it is still running: ``` - cd ../../Deploy/ + cd Deploy/ docker-compose down ``` -3. Get the lastest source code from Github: +3. Change to the parent directory of Harbor's source code folder (usually is the parent directory of `harbor/` ). Get the lastest source code from Github to overwrite the existing code: ```sh - $ git clone https://github.com/vmware/harbor + cd ../.. + git clone https://github.com/vmware/harbor ``` -4. Before upgrading Harbor, perform data migration first. +4. Before upgrading Harbor, perform database migration first. The directory **migration/** contains the tool for migration. The first step is to update values of `db_username`, `db_password`, `db_port`, `db_name` in **migration.cfg** so that they match your system's configuration. 5. The migration tool is delivered as a container, so you should build the image from its Dockerfile: ``` - cd migration/harbor-migration + cd migration/ docker build -t migrate-tool . ``` From c5492494a5d41e02eb0b17a19ba3bfa5e210699f Mon Sep 17 00:00:00 2001 From: Henry Zhang Date: Tue, 12 Jul 2016 23:19:35 +0800 Subject: [PATCH 3/8] update migration guide --- migration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration/README.md b/migration/README.md index fa400b3e3..9d75e8954 100644 --- a/migration/README.md +++ b/migration/README.md @@ -51,7 +51,7 @@ The directory **migration/** contains the tool for migration. The first step is docker run -ti --rm -v /data/database:/var/lib/mysql migrate-tool up head ``` -8. Change to `Deploy/` directory, configure Harbor by updating the file `harbor.cfg`, you may need to refer to the configuration files you backed up during step 1. Refer to [Installation & Configuration Guide ](../docs/installation_guide.md) for more info. +8. Change to `Deploy/` directory, configure Harbor by modifying the file `harbor.cfg`, you may need to refer to the configuration files you backed up during step 1. Refer to [Installation & Configuration Guide ](../docs/installation_guide.md) for more info. 9. If HTTPS has been enabled for Harbor before, restore the `nginx.conf` and key/certificate files from the backup files in Step 1. Refer to [Configuring Harbor with HTTPS Access](../docs/configure_https.md) for more info. From 314df435a342fd743b32d31c8fd71d15fa006df7 Mon Sep 17 00:00:00 2001 From: Henry Zhang Date: Tue, 12 Jul 2016 23:39:16 +0800 Subject: [PATCH 4/8] updated README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b0f2e8ad3..5ade48ef3 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,10 @@ Project Harbor is an enterprise-class registry server, which extends the open source Docker Registry server by adding the functionality usually required by an enterprise, such as security, control, and management. Harbor is primarily designed to be a private registry - providing the needed security and control that enterprises require. It also helps minimize bandwidth usage, which is helpful to both improve productivity (local network access) as well as performance (for those with poor internet connectivity). ### Features -* **Role Based Access Control**: Users and Docker repositories are organized via "projects", a user can have different permission for images under a project. +* **Role based access control**: Users and Docker repositories are organized via "projects", a user can have different permission for images under a project. +* **Image synchronization**: Images can be synchronized(replicated) between multiple registry instances. Great for load balancing and distributed data centers. * **Graphical user portal**: User can easily browse, search Docker repositories, manage projects/namespaces. +* **Garbage collection of images**: Recycle storage space by garbage collection for deleted images. * **AD/LDAP support**: Harbor integrates with existing enterprise AD/LDAP for user authentication and management. * **Auditing**: All the operations to the repositories are tracked. * **Internationalization**: Already localized for English, Chinese, German, Japanese and Russian. More languages can be added. @@ -53,7 +55,7 @@ $ docker push reg.yourdomain.com/myproject/myrepo **Data migration:** -The database schema can be updated in new version, so if you are upgrading from an older version with existing data, you need to migrate the existing data to new schema. For more details please refer to [Data Migration Guide](migration/README.md) +If you are upgrading Harbor from an older version with existing data, you may need to migrate the data to fit the new database schema. For more details please refer to [Data Migration Guide](migration/README.md) . **NOTE:** For those who don't want to clone the source, or need to install Harbor on a server not connected to the Internet - there is a pre-built installation package available. For details on how to download and use this installation package, please refer to [Installation and Configuration Guide](docs/installation_guide.md) . From e3df311c0c77cf7d35f54c3bc3b7c2daeaa183a6 Mon Sep 17 00:00:00 2001 From: Henry Zhang Date: Wed, 13 Jul 2016 10:55:53 +0800 Subject: [PATCH 5/8] update README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ade48ef3..e9083c10f 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,8 @@ Project Harbor is an enterprise-class registry server, which extends the open so ### Features * **Role based access control**: Users and Docker repositories are organized via "projects", a user can have different permission for images under a project. -* **Image synchronization**: Images can be synchronized(replicated) between multiple registry instances. Great for load balancing and distributed data centers. +* **Image replication**: Images can be replicated(synchronized) between multiple registry instances. Great for load balancing and distributed data centers. * **Graphical user portal**: User can easily browse, search Docker repositories, manage projects/namespaces. -* **Garbage collection of images**: Recycle storage space by garbage collection for deleted images. * **AD/LDAP support**: Harbor integrates with existing enterprise AD/LDAP for user authentication and management. * **Auditing**: All the operations to the repositories are tracked. * **Internationalization**: Already localized for English, Chinese, German, Japanese and Russian. More languages can be added. From c0ab2dbdfc2607b4f1b3c9e100b6d17924c567e5 Mon Sep 17 00:00:00 2001 From: Henry Zhang Date: Wed, 13 Jul 2016 15:42:47 +0800 Subject: [PATCH 6/8] update readme, migration guide --- README.md | 4 +- .../README.md => docs/migration_guide.md | 58 +++++++++++++------ 2 files changed, 43 insertions(+), 19 deletions(-) rename migration/README.md => docs/migration_guide.md (67%) diff --git a/README.md b/README.md index e9083c10f..ad0817a4e 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Harbor only works with docker 1.10+ and docker-compose 1.6.0+, and an internet-c Generated configuration file: ./config/registry/config.yml Generated configuration file: ./config/db/env - $ docker-compose up + $ docker-compose up -d ``` _If everything worked properly, you should be able to open a browser to visit the admin portal at http://reg.yourdomain.com . Note that the default administrator username/password are admin/Harbor12345 ._ @@ -54,7 +54,7 @@ $ docker push reg.yourdomain.com/myproject/myrepo **Data migration:** -If you are upgrading Harbor from an older version with existing data, you may need to migrate the data to fit the new database schema. For more details please refer to [Data Migration Guide](migration/README.md) . +If you are upgrading Harbor from an older version with existing data, you may need to migrate the data to fit the new database schema. For more details please refer to [Data Migration Guide](docs/migration_guide.md) . **NOTE:** For those who don't want to clone the source, or need to install Harbor on a server not connected to the Internet - there is a pre-built installation package available. For details on how to download and use this installation package, please refer to [Installation and Configuration Guide](docs/installation_guide.md) . diff --git a/migration/README.md b/docs/migration_guide.md similarity index 67% rename from migration/README.md rename to docs/migration_guide.md index 9d75e8954..743ce67da 100644 --- a/migration/README.md +++ b/docs/migration_guide.md @@ -9,23 +9,21 @@ When upgrading your existing Habor instance to a newer version, you may need to ### Upgrading Harbor and migrating data -1. Log in to the machine that Harbor runs on, back up Harbor's configuration files. - ```sh - mkdir -p /tmp/harbor/config - cp -r Deploy/config /tmp/harbor/config - cp Deploy/harbor.cfg /tmp/harbor - ``` - -2. Next, stop and remove existing Harbor service if it is still running: +1. Log in to the machine that Harbor runs on, stop and remove existing Harbor service if it is still running: ``` cd Deploy/ docker-compose down ``` -3. Change to the parent directory of Harbor's source code folder (usually is the parent directory of `harbor/` ). Get the lastest source code from Github to overwrite the existing code: +2. Back up Harbor's current source code so that you can roll back to the current version when it is necessary. ```sh cd ../.. + mv harbor /tmp/harbor + ``` + +3. Get the lastest source code from Github: + ```sh git clone https://github.com/vmware/harbor ``` @@ -51,11 +49,11 @@ The directory **migration/** contains the tool for migration. The first step is docker run -ti --rm -v /data/database:/var/lib/mysql migrate-tool up head ``` -8. Change to `Deploy/` directory, configure Harbor by modifying the file `harbor.cfg`, you may need to refer to the configuration files you backed up during step 1. Refer to [Installation & Configuration Guide ](../docs/installation_guide.md) for more info. +8. Change to `Deploy/` directory, configure Harbor by modifying the file `harbor.cfg`, you may need to refer to the configuration files you've backed up during step 2. Refer to [Installation & Configuration Guide ](../docs/installation_guide.md) for more info. -9. If HTTPS has been enabled for Harbor before, restore the `nginx.conf` and key/certificate files from the backup files in Step 1. Refer to [Configuring Harbor with HTTPS Access](../docs/configure_https.md) for more info. +9. If HTTPS has been enabled for Harbor before, restore the `nginx.conf` and key/certificate files from the backup files in Step 2. Refer to [Configuring Harbor with HTTPS Access](../docs/configure_https.md) for more info. -10. Run the `./prepare` script to generate necessary config files. +10. Under the directory `Deploy/`, run the `./prepare` script to generate necessary config files. 11. Rebuild Harbor and restart the registry service @@ -63,6 +61,37 @@ The directory **migration/** contains the tool for migration. The first step is docker-compose up --build -d ``` +### Roll back from an upgrade +For any reason, if you want to roll back to the previous version of Harbor, follow the below steps: + +1. Stop and remove the current Harbor service if it is still running. + + ``` + cd Deploy/ + docker-compose down + ``` +2. Restore database from backup file in `/path/to/backup` . + + ``` + docker run -ti --rm -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup migrate-tool restore + ``` + +3. Remove current source code of Harbor. + ``` + rm -rf harbor + ``` + +4. Restore the source code of an older version of Harbor. + ```sh + mv /tmp/harbor harbor + ``` + +5. Restart Harbor service using the previous configuration. + ```sh + cd Deploy/ + docker-compose up --build -d + ``` + ### Migration tool reference - Use `help` command to show instructions of the migration tool: @@ -72,8 +101,3 @@ The directory **migration/** contains the tool for migration. The first step is ```docker run --rm -v /data/database:/var/lib/mysql migrate-tool test``` -- Restore database from backup file in `/path/to/backup` - - ``` - docker run -ti --rm -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup migrate-tool restore - ``` From 67a8c8631ac62a7f4d6c2d66d28a67b1cb49c026 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 13 Jul 2016 14:35:52 +0800 Subject: [PATCH 7/8] fix #520 #521 #522 --- api/target.go | 14 +++++++------- dao/dao_test.go | 16 ++++++++++++++++ dao/replication_job.go | 9 ++++----- job/replication/transfer.go | 17 ++++++++++++----- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/api/target.go b/api/target.go index c89f1a3fd..1c78b915d 100644 --- a/api/target.go +++ b/api/target.go @@ -190,14 +190,14 @@ func (t *TargetAPI) Post() { t.CustomAbort(http.StatusConflict, "name is already used") } - ta, err = dao.GetRepTargetByConnInfo(target.URL, target.Username) + ta, err = dao.GetRepTargetByEndpoint(target.URL) if err != nil { - log.Errorf("failed to get target [ %s %s ]: %v", target.URL, target.Username, err) + log.Errorf("failed to get target [ %s ]: %v", target.URL, err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } if ta != nil { - t.CustomAbort(http.StatusConflict, "the connection information[ endpoint, username ] is conflict with other target") + t.CustomAbort(http.StatusConflict, fmt.Sprintf("the target whose endpoint is %s already exists", target.URL)) } if len(target.Password) != 0 { @@ -260,15 +260,15 @@ func (t *TargetAPI) Put() { } } - if target.URL != originalTarget.URL || target.Username != originalTarget.Username { - ta, err := dao.GetRepTargetByConnInfo(target.URL, target.Username) + if target.URL != originalTarget.URL { + ta, err := dao.GetRepTargetByEndpoint(target.URL) if err != nil { - log.Errorf("failed to get target [ %s %s ]: %v", target.URL, target.Username, err) + log.Errorf("failed to get target [ %s ]: %v", target.URL, err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } if ta != nil { - t.CustomAbort(http.StatusConflict, "the connection information[ endpoint, username ] is conflict with other target") + t.CustomAbort(http.StatusConflict, fmt.Sprintf("the target whose endpoint is %s already exists", target.URL)) } } diff --git a/dao/dao_test.go b/dao/dao_test.go index 699224700..3d375927d 100644 --- a/dao/dao_test.go +++ b/dao/dao_test.go @@ -855,6 +855,22 @@ func TestGetRepTargetByName(t *testing.T) { } } +func TestGetRepTargetByEndpoint(t *testing.T) { + target, err := GetRepTarget(targetID) + if err != nil { + t.Fatalf("failed to get target %d: %v", targetID, err) + } + + target2, err := GetRepTargetByEndpoint(target.URL) + if err != nil { + t.Fatalf("failed to get target %s: %v", target.URL, err) + } + + if target.URL != target2.URL { + t.Errorf("unexpected target URL: %s, expected: %s", target2.URL, target.URL) + } +} + func TestUpdateRepTarget(t *testing.T) { target := &models.RepTarget{ Name: "name", diff --git a/dao/replication_job.go b/dao/replication_job.go index adc47391c..1c5a31cbc 100644 --- a/dao/replication_job.go +++ b/dao/replication_job.go @@ -53,14 +53,13 @@ func GetRepTargetByName(name string) (*models.RepTarget, error) { return &t, err } -// GetRepTargetByConnInfo ... -func GetRepTargetByConnInfo(endpoint, username string) (*models.RepTarget, error) { +// GetRepTargetByEndpoint ... +func GetRepTargetByEndpoint(endpoint string) (*models.RepTarget, error) { o := GetOrmer() t := models.RepTarget{ - URL: endpoint, - Username: username, + URL: endpoint, } - err := o.Read(&t, "URL", "Username") + err := o.Read(&t, "URL") if err == orm.ErrNoRows { return nil, nil } diff --git a/job/replication/transfer.go b/job/replication/transfer.go index 8ac3015bd..506f35477 100644 --- a/job/replication/transfer.go +++ b/job/replication/transfer.go @@ -28,6 +28,7 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" + "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry" @@ -192,7 +193,13 @@ enter: return "", err } if !exist { - err := c.createProject() + project, err := dao.GetProjectByName(c.project) + if err != nil { + c.logger.Errorf("an error occurred while getting project %s on %s: %v", c.project, c.srcURL, err) + return "", err + } + + err = c.createProject(project.Public == 1) if err != nil { // other job may be also doing the same thing when the current job // is creating project, so when the response code is 409, re-check @@ -286,13 +293,13 @@ func (c *Checker) projectExist() (exist, canWrite bool, err error) { return } -func (c *Checker) createProject() error { - // TODO handle publicity of project +func (c *Checker) createProject(isPublic bool) error { project := struct { ProjectName string `json:"project_name"` Public bool `json:"public"` }{ ProjectName: c.project, + Public: isPublic, } data, err := json.Marshal(project) @@ -495,8 +502,8 @@ func (m *ManifestPusher) enter() (string, error) { } else { m.logger.Infof("manifest of %s:%s exists on source registry %s, continue manifest pushing", name, tag, m.srcURL) - _, manifestExist, err := m.dstClient.ManifestExist(m.digest) - if manifestExist { + digest, manifestExist, err := m.dstClient.ManifestExist(tag) + if manifestExist && digest == m.digest { m.logger.Infof("manifest of %s:%s exists on destination registry %s, skip manifest pushing", name, tag, m.dstURL) m.tags = m.tags[1:] From f685702475ae2637f769aadb1cdc943fffc86daa Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 13 Jul 2016 17:19:23 +0800 Subject: [PATCH 8/8] do not trigger replication if the policy is disabled --- api/utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/utils.go b/api/utils.go index e0f5d13cc..3aafebb1c 100644 --- a/api/utils.go +++ b/api/utils.go @@ -162,6 +162,9 @@ func TriggerReplicationByRepository(repository string, tags []string, operation } for _, policy := range policies { + if policy.Enabled == 0 { + continue + } if err := TriggerReplication(policy.ID, repository, tags, operation); err != nil { log.Errorf("failed to trigger replication of policy %d for %s: %v", policy.ID, repository, err) } else {