diff --git a/.gitignore b/.gitignore index 6b04c4ad3..b53274b48 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ Deploy/config/ui/app.conf Deploy/config/db/env Deploy/harbor.cfg ui/ui +*.pyc diff --git a/Deploy/db/registry.sql b/Deploy/db/registry.sql index 522b2b012..0380e739c 100644 --- a/Deploy/db/registry.sql +++ b/Deploy/db/registry.sql @@ -103,19 +103,41 @@ create table access_log ( FOREIGN KEY (project_id) REFERENCES project (project_id) ); -create table job ( - job_id int NOT NULL AUTO_INCREMENT, - job_type varchar(64) NOT NULL, - status varchar(64) NOT NULL, - options text, - parms text, +create table replication_policy ( + id int NOT NULL AUTO_INCREMENT, + name varchar(256), + project_id int NOT NULL, + target_id int NOT NULL, enabled tinyint(1) NOT NULL DEFAULT 1, + description text, cron_str varchar(256), - triggered_by varchar(64), - creation_time timestamp, - update_time timestamp, - PRIMARY KEY (job_id) -); + start_time timestamp, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + PRIMARY KEY (id) + ); + +create table replication_target ( + id int NOT NULL AUTO_INCREMENT, + name varchar(64), + url varchar(64), + username varchar(40), + password varchar(40), + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + PRIMARY KEY (id) + ); + +create table replication_job ( + id int NOT NULL AUTO_INCREMENT, + status varchar(64) NOT NULL, + policy_id int NOT NULL, + repository varchar(256) NOT NULL, + operation varchar(64) NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + PRIMARY KEY (id) + ); create table job_log ( log_id int NOT NULL AUTO_INCREMENT, @@ -125,7 +147,7 @@ create table job_log ( creation_time timestamp, update_time timestamp, PRIMARY KEY (log_id), - FOREIGN KEY (job_id) REFERENCES job (job_id) + FOREIGN KEY (job_id) REFERENCES replication_job (id) ); create table properties ( @@ -136,3 +158,9 @@ create table properties ( insert into properties (k, v) values ('schema_version', '0.1.1'); + +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +insert into alembic_version values ('0.1.1'); diff --git a/Deploy/prepare b/Deploy/prepare index 33288d06e..edd31ea95 100755 --- a/Deploy/prepare +++ b/Deploy/prepare @@ -116,7 +116,7 @@ FNULL = open(os.devnull, 'w') from functools import wraps def stat_decorator(func): - #@wraps(func) + @wraps(func) def check_wrapper(*args, **kwargs): stat = func(*args, **kwargs) message = "Generated configuration file: %s" % kwargs['path'] \ diff --git a/api/job.go b/api/job.go index 5d5066a5f..48930b7ef 100644 --- a/api/job.go +++ b/api/job.go @@ -1,5 +1,6 @@ package api +/* import ( "encoding/json" "fmt" @@ -15,6 +16,7 @@ type JobAPI struct { BaseAPI } + func (ja *JobAPI) Post() { var je models.JobEntry ja.DecodeJSONReq(&je) @@ -82,4 +84,4 @@ func (ja *JobAPI) Get() { ja.Data["json"] = jobs } ja.ServeJSON() -} +}*/ diff --git a/api/replication.go b/api/replication.go new file mode 100644 index 000000000..ba6f81471 --- /dev/null +++ b/api/replication.go @@ -0,0 +1,106 @@ +package api + +import ( + "encoding/json" + "fmt" + "github.com/vmware/harbor/dao" + "github.com/vmware/harbor/job" + "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" + "io/ioutil" + "net/http" + "net/http/httputil" + "os" + "strconv" + "strings" +) + +type ReplicationJob struct { + BaseAPI +} + +type ReplicationReq struct { + PolicyID int64 `json:"policy_id"` +} + +func (rj *ReplicationJob) Post() { + var data ReplicationReq + rj.DecodeJSONReq(&data) + log.Debugf("data: %+v", data) + p, err := dao.GetRepPolicy(data.PolicyID) + if err != nil { + log.Errorf("Failed to get policy, error: %v", err) + rj.RenderError(http.StatusInternalServerError, fmt.Sprintf("Failed to get policy, id: %d", data.PolicyID)) + return + } + repoList, err := getRepoList(p.ProjectID) + if err != nil { + log.Errorf("Failed to get repository list, project id: %d, error: %v", p.ProjectID, err) + rj.RenderError(http.StatusInternalServerError, err.Error()) + return + } + log.Debugf("repo list: %v", repoList) + for _, repo := range repoList { + j := models.RepJob{ + Repository: repo, + PolicyID: data.PolicyID, + Operation: models.RepOpTransfer, + } + log.Debugf("Creating job for repo: %s, policy: %d", repo, data.PolicyID) + id, err := dao.AddRepJob(j) + if err != nil { + log.Errorf("Failed to insert job record, error: %v", err) + rj.RenderError(http.StatusInternalServerError, err.Error()) + return + } + log.Debugf("Send job to scheduler, job id: %d", id) + job.Schedule(id) + } +} + +// calls the api from UI to get repo list +func getRepoList(projectID int64) ([]string, error) { + uiURL := os.Getenv("UI_URL") + if len(uiURL) == 0 { + uiURL = "ui" + } + if !strings.HasSuffix(uiURL, "/") { + uiURL += "/" + } + //TODO:Use secret key instead + uiUser := os.Getenv("UI_USR") + if len(uiUser) == 0 { + uiUser = "admin" + } + uiPwd := os.Getenv("UI_PWD") + if len(uiPwd) == 0 { + uiPwd = "Harbor12345" + } + client := &http.Client{} + req, err := http.NewRequest("GET", uiURL+"api/repositories?project_id="+strconv.Itoa(int(projectID)), nil) + if err != nil { + log.Errorf("Error when creating request: %v") + return nil, err + } + req.SetBasicAuth(uiUser, uiPwd) + resp, err := client.Do(req) + if err != nil { + log.Errorf("Error when calling UI api to get repositories, error: %v", err) + return nil, err + } + if resp.StatusCode != http.StatusOK { + log.Errorf("Unexpected status code: %d", resp.StatusCode) + dump, _ := httputil.DumpResponse(resp, true) + log.Debugf("response: %q", dump) + return nil, fmt.Errorf("Unexpected status code when getting repository list: %d", resp.StatusCode) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Errorf("Failed to read the response body, error: %v") + return nil, err + } + var repoList []string + err = json.Unmarshal(body, &repoList) + return repoList, err +} diff --git a/api/user.go b/api/user.go index c9bb99800..45869fb6c 100644 --- a/api/user.go +++ b/api/user.go @@ -187,7 +187,9 @@ func (ua *UserAPI) Delete() { // ChangePassword handles PUT to /api/users/{}/password func (ua *UserAPI) ChangePassword() { - if !(ua.AuthMode == "db_auth") { + ldapAdminUser := (ua.AuthMode == "ldap_auth" && ua.userID == 1 && ua.userID == ua.currentUserID) + + if !(ua.AuthMode == "db_auth" || ldapAdminUser) { ua.CustomAbort(http.StatusForbidden, "") } diff --git a/controllers/base.go b/controllers/base.go index e76abe4be..7d4b3e535 100644 --- a/controllers/base.go +++ b/controllers/base.go @@ -41,6 +41,7 @@ type BaseController struct { beego.Controller i18n.Locale SelfRegistration bool + IsLdapAdminUser bool IsAdmin bool AuthMode string } @@ -115,7 +116,11 @@ func (b *BaseController) Prepare() { if sessionUserID != nil { b.Data["Username"] = b.GetSession("username") b.Data["UserId"] = sessionUserID.(int) - + + if (sessionUserID == 1 && b.AuthMode == "ldap_auth") { + b.IsLdapAdminUser = true + } + var err error b.IsAdmin, err = dao.IsAdminRole(sessionUserID.(int)) if err != nil { @@ -126,6 +131,7 @@ func (b *BaseController) Prepare() { b.Data["IsAdmin"] = b.IsAdmin b.Data["SelfRegistration"] = b.SelfRegistration + b.Data["IsLdapAdminUser"] = b.IsLdapAdminUser } diff --git a/dao/dao_test.go b/dao/dao_test.go index 99ece3348..30b9d7961 100644 --- a/dao/dao_test.go +++ b/dao/dao_test.go @@ -20,11 +20,9 @@ import ( "testing" "time" - "github.com/vmware/harbor/utils/log" - - "github.com/vmware/harbor/models" - "github.com/astaxie/beego/orm" + "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" ) func execUpdate(o orm.Ormer, sql string, params interface{}) error { @@ -722,3 +720,227 @@ func TestDeleteUser(t *testing.T) { t.Errorf("user is not nil after deletion, user: %+v", user) } } + +var targetID, policyID, jobID int64 + +func TestAddRepTarget(t *testing.T) { + target := models.RepTarget{ + URL: "127.0.0.1:5000", + Username: "admin", + Password: "admin", + } + //_, err := AddRepTarget(target) + id, err := AddRepTarget(target) + t.Logf("added target, id: %d", id) + if err != nil { + t.Errorf("Error occurred in AddRepTarget: %v", err) + } else { + targetID = id + } + id2 := id + 99 + tgt, err := GetRepTarget(id2) + if err != nil { + t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id2) + } + if tgt != nil { + t.Errorf("There should not be a target with id: %d", id2) + } + tgt, err = GetRepTarget(id) + if err != nil { + t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id) + } + if tgt == nil { + t.Errorf("Unable to find a target with id: %d", id) + } + if tgt.URL != "127.0.0.1:5000" { + t.Errorf("Unexpected url in target: %s, expected 127.0.0.1:5000", tgt.URL) + } + if tgt.Username != "admin" { + t.Errorf("Unexpected username in target: %s, expected admin", tgt.Username) + } +} + +func TestAddRepPolicy(t *testing.T) { + policy := models.RepPolicy{ + ProjectID: 1, + Enabled: 1, + TargetID: targetID, + Description: "whatever", + Name: "mypolicy", + } + id, err := AddRepPolicy(policy) + t.Logf("added policy, id: %d", id) + if err != nil { + t.Errorf("Error occurred in AddRepPolicy: %v", err) + } else { + policyID = id + } + p, err := GetRepPolicy(id) + if err != nil { + t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, id) + } + if p == nil { + t.Errorf("Unable to find a policy with id: %d", id) + } + + if p.Name != "mypolicy" || p.TargetID != targetID || p.Enabled != 1 || p.Description != "whatever" { + t.Errorf("The data does not match, expected: Name: mypolicy, TargetID: %d, Enabled: 1, Description: whatever;\n result: Name: %s, TargetID: %d, Enabled: %d, Description: %s", + targetID, p.Name, p.TargetID, p.Enabled, p.Description) + } + var tm time.Time = time.Now().AddDate(0, 0, -1) + if !p.StartTime.After(tm) { + t.Errorf("Unexpected start_time: %v", p.StartTime) + } + +} + +func TestDisableRepPolicy(t *testing.T) { + err := DisableRepPolicy(policyID) + if err != nil { + t.Errorf("Failed to disable policy, id: %d", policyID) + } + p, err := GetRepPolicy(policyID) + if err != nil { + t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, policyID) + } + if p == nil { + t.Errorf("Unable to find a policy with id: %d", policyID) + } + if p.Enabled == 1 { + t.Errorf("The Enabled value of replication policy is still 1 after disabled, id: %d", policyID) + } +} + +func TestEnableRepPolicy(t *testing.T) { + err := EnableRepPolicy(policyID) + if err != nil { + t.Errorf("Failed to disable policy, id: %d", policyID) + } + p, err := GetRepPolicy(policyID) + if err != nil { + t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, policyID) + } + if p == nil { + t.Errorf("Unable to find a policy with id: %d", policyID) + } + if p.Enabled == 0 { + t.Errorf("The Enabled value of replication policy is still 0 after disabled, id: %d", policyID) + } +} + +func TestAddRepPolicy2(t *testing.T) { + policy2 := models.RepPolicy{ + ProjectID: 3, + Enabled: 0, + TargetID: 3, + Description: "whatever", + Name: "mypolicy", + } + id, err := AddRepPolicy(policy2) + t.Logf("added policy, id: %d", id) + if err != nil { + t.Errorf("Error occurred in AddRepPolicy: %v", err) + } + p, err := GetRepPolicy(id) + if err != nil { + t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, id) + } + if p == nil { + t.Errorf("Unable to find a policy with id: %d", id) + } + var tm time.Time + if p.StartTime.After(tm) { + t.Errorf("Unexpected start_time: %v", p.StartTime) + } +} + +func TestAddRepJob(t *testing.T) { + job := models.RepJob{ + Repository: "library/ubuntu", + PolicyID: policyID, + Operation: "transfer", + } + id, err := AddRepJob(job) + if err != nil { + t.Errorf("Error occurred in AddRepJob: %v", err) + } else { + jobID = id + } + j, err := GetRepJob(id) + if err != nil { + t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, id) + } + if j == nil { + t.Errorf("Unable to find a job with id: %d", id) + } + if j.Status != models.JobPending || j.Repository != "library/ubuntu" || j.PolicyID != policyID || j.Operation != "transfer" { + t.Errorf("Expected data of job, id: %d, Status: %s, Repository: library/ubuntu, PolicyID: %d, Operation: transfer, "+ + "but in returned data:, Status: %s, Repository: %s, Operation: %s, PolicyID: %d", id, models.JobPending, policyID, j.Status, j.Repository, j.Operation, j.PolicyID) + } +} + +func TestUpdateRepJobStatus(t *testing.T) { + err := UpdateRepJobStatus(jobID, models.JobFinished) + if err != nil { + t.Errorf("Error occured in UpdateRepJobStatus, error: %v, id: %d", err, jobID) + return + } + j, err := GetRepJob(jobID) + if err != nil { + t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, jobID) + } + if j == nil { + t.Errorf("Unable to find a job with id: %d", jobID) + } + if j.Status != models.JobFinished { + t.Errorf("Job's status: %s, expected: %s, id: %d", j.Status, models.JobFinished, jobID) + } +} + +func TestDeleteRepTarget(t *testing.T) { + err := DeleteRepTarget(targetID) + if err != nil { + t.Errorf("Error occured in DeleteRepTarget: %v, id: %d", err, targetID) + return + } + t.Logf("deleted target, id: %d", targetID) + tgt, err := GetRepTarget(targetID) + if err != nil { + t.Errorf("Error occurred in GetTarget: %v, id: %d", err, targetID) + } + if tgt != nil { + t.Errorf("Able to find target after deletion, id: %d", targetID) + } +} + +func TestDeleteRepPolicy(t *testing.T) { + err := DeleteRepPolicy(policyID) + if err != nil { + t.Errorf("Error occured in DeleteRepPolicy: %v, id: %d", err, policyID) + return + } + t.Logf("delete rep policy, id: %d", policyID) + p, err := GetRepPolicy(policyID) + if err != nil { + t.Errorf("Error occured in GetRepPolicy:%v", err) + } + if p != nil { + t.Errorf("Able to find rep policy after deletion, id: %d", policyID) + } +} + +func TestDeleteRepJob(t *testing.T) { + err := DeleteRepJob(jobID) + if err != nil { + t.Errorf("Error occured in DeleteRepJob: %v, id: %d", err, jobID) + return + } + t.Logf("deleted rep job, id: %d", jobID) + j, err := GetRepJob(jobID) + if err != nil { + t.Errorf("Error occured in GetRepJob:%v", err) + } + if j != nil { + t.Errorf("Able to find rep job after deletion, id: %d", jobID) + } +} diff --git a/dao/replication_job.go b/dao/replication_job.go new file mode 100644 index 000000000..695464621 --- /dev/null +++ b/dao/replication_job.go @@ -0,0 +1,113 @@ +package dao + +import ( + "fmt" + "github.com/astaxie/beego/orm" + "github.com/vmware/harbor/models" +) + +func AddRepTarget(target models.RepTarget) (int64, error) { + o := orm.NewOrm() + return o.Insert(&target) +} +func GetRepTarget(id int64) (*models.RepTarget, error) { + o := orm.NewOrm() + t := models.RepTarget{ID: id} + err := o.Read(&t) + if err == orm.ErrNoRows { + return nil, nil + } + return &t, err +} +func DeleteRepTarget(id int64) error { + o := orm.NewOrm() + _, err := o.Delete(&models.RepTarget{ID: id}) + return err +} + +func AddRepPolicy(policy models.RepPolicy) (int64, error) { + o := orm.NewOrm() + sqlTpl := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time ) values (?, ?, ?, ?, ?, ?, %s, NOW(), NOW())` + var sql string + if policy.Enabled == 1 { + sql = fmt.Sprintf(sqlTpl, "NOW()") + } else { + sql = fmt.Sprintf(sqlTpl, "NULL") + } + p, err := o.Raw(sql).Prepare() + if err != nil { + return 0, err + } + r, err := p.Exec(policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.CronStr) + if err != nil { + return 0, err + } + id, err := r.LastInsertId() + return id, err +} +func GetRepPolicy(id int64) (*models.RepPolicy, error) { + o := orm.NewOrm() + p := models.RepPolicy{ID: id} + err := o.Read(&p) + if err == orm.ErrNoRows { + return nil, nil + } + return &p, err +} +func DeleteRepPolicy(id int64) error { + o := orm.NewOrm() + _, err := o.Delete(&models.RepPolicy{ID: id}) + return err +} +func updateRepPolicyEnablement(id int64, enabled int) error { + o := orm.NewOrm() + p := models.RepPolicy{ + ID: id, + Enabled: enabled} + num, err := o.Update(&p, "Enabled") + if num == 0 { + err = fmt.Errorf("Failed to update replication policy with id: %d", id) + } + return err +} +func EnableRepPolicy(id int64) error { + return updateRepPolicyEnablement(id, 1) +} + +func DisableRepPolicy(id int64) error { + return updateRepPolicyEnablement(id, 0) +} + +func AddRepJob(job models.RepJob) (int64, error) { + o := orm.NewOrm() + if len(job.Status) == 0 { + job.Status = models.JobPending + } + return o.Insert(&job) +} +func GetRepJob(id int64) (*models.RepJob, error) { + o := orm.NewOrm() + j := models.RepJob{ID: id} + err := o.Read(&j) + if err == orm.ErrNoRows { + return nil, nil + } + return &j, err +} +func DeleteRepJob(id int64) error { + o := orm.NewOrm() + _, err := o.Delete(&models.RepJob{ID: id}) + return err +} +func UpdateRepJobStatus(id int64, status string) error { + o := orm.NewOrm() + j := models.RepJob{ + ID: id, + Status: status, + } + num, err := o.Update(&j, "Status") + if num == 0 { + err = fmt.Errorf("Failed to update replication job with id: %d", id) + } + return err +} diff --git a/job/imgout/runner.go b/job/imgout/runner.go index 54097ee33..d4311fb4a 100644 --- a/job/imgout/runner.go +++ b/job/imgout/runner.go @@ -1,5 +1,6 @@ package imgout +/* import ( "encoding/json" //"github.com/vmware/harbor/dao" @@ -72,3 +73,4 @@ func (r *Runner) init(je models.JobEntry) error { r.AddTransition("push-img", job.JobFinished, job.StatusUpdater{job.DummyHandler{JobID: r.JobID}, job.JobFinished}) return nil } +*/ diff --git a/job/scheduler.go b/job/scheduler.go index bb265e80a..80d9859fa 100644 --- a/job/scheduler.go +++ b/job/scheduler.go @@ -1,32 +1,31 @@ package job import ( + "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" - "os" - "strconv" ) -var lock chan bool +func Schedule(jobID int64) { + //TODO: introduce jobqueue to better control concurrent job numbers + go HandleRepJob(jobID) +} -const defaultMaxJobs int64 = 10 - -func init() { - maxJobsEnv := os.Getenv("MAX_CONCURRENT_JOB") - maxJobs, err := strconv.ParseInt(maxJobsEnv, 10, 32) +func HandleRepJob(id int64) { + sm := &JobSM{JobID: id} + err := sm.Init() if err != nil { - log.Warningf("Failed to parse max job setting, error: %v, the default value: %d will be used", err, defaultMaxJobs) - maxJobs = defaultMaxJobs + log.Errorf("Failed to initialize statemachine, error: %v") + err2 := dao.UpdateRepJobStatus(id, models.JobError) + if err2 != nil { + log.Errorf("Failed to update job status to ERROR, error:%v", err2) + } + return + } + if sm.Parms.Enabled == 0 { + log.Debugf("The policy of job:%d is disabled, will cancel the job") + _ = dao.UpdateRepJobStatus(id, models.JobCanceled) + } else { + sm.Start(models.JobRunning) } - lock = make(chan bool, maxJobs) -} -func Schedule(job models.JobEntry) { - log.Infof("job: %d will be scheduled", job.ID) - //TODO: add support for cron string when needed. - go func() { - lock <- true - defer func() { <-lock }() - log.Infof("running job: %d", job.ID) - run(job) - }() } diff --git a/job/statehandlers.go b/job/statehandlers.go new file mode 100644 index 000000000..9f68ff9bc --- /dev/null +++ b/job/statehandlers.go @@ -0,0 +1,73 @@ +package job + +import ( + "github.com/vmware/harbor/dao" + "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" + "time" +) + +// StateHandler handles transition, it associates with each state, will be called when +// SM enters and exits a state during a transition. +type StateHandler interface { + // Enter returns the next state, if it returns empty string the SM will hold the current state or + // or decide the next state. + Enter() (string, error) + //Exit should be idempotent + Exit() error +} + +type DummyHandler struct { + JobID int64 +} + +func (dh DummyHandler) Enter() (string, error) { + return "", nil +} + +func (dh DummyHandler) Exit() error { + return nil +} + +type StatusUpdater struct { + DummyHandler + State string +} + +func (su StatusUpdater) Enter() (string, error) { + err := dao.UpdateRepJobStatus(su.JobID, su.State) + if err != nil { + log.Warningf("Failed to update state of job: %d, state: %s, error: %v", su.JobID, su.State, err) + } + var next string = models.JobContinue + if su.State == models.JobStopped || su.State == models.JobError || su.State == models.JobFinished { + next = "" + } + return next, err +} + +type ImgPuller struct { + DummyHandler + img string + logger Logger +} + +func (ip ImgPuller) Enter() (string, error) { + ip.logger.Infof("I'm pretending to pull img:%s, then sleep 30s", ip.img) + time.Sleep(30 * time.Second) + ip.logger.Infof("wake up from sleep....") + return "push-img", nil +} + +type ImgPusher struct { + DummyHandler + targetURL string + logger Logger +} + +func (ip ImgPusher) Enter() (string, error) { + ip.logger.Infof("I'm pretending to push img to:%s, then sleep 30s", ip.targetURL) + time.Sleep(30 * time.Second) + ip.logger.Infof("wake up from sleep....") + return models.JobContinue, nil +} diff --git a/job/statemachine.go b/job/statemachine.go index f6bc88b92..f4056cad9 100644 --- a/job/statemachine.go +++ b/job/statemachine.go @@ -3,59 +3,22 @@ package job import ( "fmt" "github.com/vmware/harbor/dao" + "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" + "os" "sync" ) -// StateHandler handles transition, it associates with each state, will be called when -// SM enters and exits a state during a transition. -type StateHandler interface { - // Enter returns the next state, if it returns empty string the SM will hold the current state or - // or decide the next state. - Enter() (string, error) - //Exit should be idempotent - Exit() error +type RepJobParm struct { + LocalRegURL string + TargetURL string + TargetUsername string + TargetPassword string + Repository string + Enabled int + Operation string } -type DummyHandler struct { - JobID int64 -} - -func (dh DummyHandler) Enter() (string, error) { - return "", nil -} - -func (dh DummyHandler) Exit() error { - return nil -} - -type StatusUpdater struct { - DummyHandler - State string -} - -func (su StatusUpdater) Enter() (string, error) { - err := dao.UpdateJobStatus(su.JobID, su.State) - if err != nil { - log.Warningf("Failed to update state of job: %d, state: %s, error: %v", su.JobID, su.State, err) - } - var next string = JobContinue - if su.State == JobStopped || su.State == JobError || su.State == JobFinished { - next = "" - } - return next, err -} - -const ( - JobPending string = "pending" - JobRunning string = "running" - JobError string = "error" - JobStopped string = "stopped" - JobFinished string = "finished" - // statemachine will move to next possible state based on trasition table - JobContinue string = "_continue" -) - type JobSM struct { JobID int64 CurrentState string @@ -64,8 +27,10 @@ type JobSM struct { ForcedStates map[string]struct{} Transitions map[string]map[string]struct{} Handlers map[string]StateHandler - lock *sync.Mutex desiredState string + Logger Logger + Parms *RepJobParm + lock *sync.Mutex } // EnsterState transit the statemachine from the current state to the state in parameter. @@ -87,7 +52,7 @@ func (sm *JobSM) EnterState(s string) (string, error) { log.Debugf("No handler found for state:%s, skip", sm.CurrentState) } enterHandler, ok := sm.Handlers[s] - var next string = JobContinue + var next string = models.JobContinue var err error if ok { if next, err = enterHandler.Enter(); err != nil { @@ -115,14 +80,14 @@ func (sm *JobSM) Start(s string) { sm.setDesiredState("") continue } - if n == JobContinue && len(sm.Transitions[sm.CurrentState]) == 1 { + if n == models.JobContinue && len(sm.Transitions[sm.CurrentState]) == 1 { for n = range sm.Transitions[sm.CurrentState] { break } log.Debugf("Continue to state: %s", n) continue } - if n == JobContinue && len(sm.Transitions[sm.CurrentState]) != 1 { + if n == models.JobContinue && len(sm.Transitions[sm.CurrentState]) != 1 { log.Errorf("Next state is continue but there are %d possible next states in transition table", len(sm.Transitions[sm.CurrentState])) err = fmt.Errorf("Unable to continue") break @@ -132,7 +97,7 @@ func (sm *JobSM) Start(s string) { } if err != nil { log.Warningf("The statemachin will enter error state due to error: %v", err) - sm.EnterState(JobError) + sm.EnterState(models.JobError) } } @@ -154,7 +119,7 @@ func (sm *JobSM) RemoveTransition(from string, to string) { } func (sm *JobSM) Stop() { - sm.setDesiredState(JobStopped) + sm.setDesiredState(models.JobStopped) } func (sm *JobSM) getDesiredState() string { @@ -169,12 +134,60 @@ func (sm *JobSM) setDesiredState(s string) { sm.desiredState = s } -func (sm *JobSM) InitJobSM() { +func (sm *JobSM) Init() error { + //init parms + regURL := os.Getenv("LOCAL_REGISTRY_URL") + if len(regURL) == 0 { + regURL = "http://registry:5000/" + } + job, err := dao.GetRepJob(sm.JobID) + if err != nil { + return fmt.Errorf("Failed to get job, error: %v", err) + } + if job == nil { + return fmt.Errorf("The job doesn't exist in DB, job id: %d", sm.JobID) + } + policy, err := dao.GetRepPolicy(job.PolicyID) + if err != nil { + return fmt.Errorf("Failed to get policy, error: %v", err) + } + if policy == nil { + return fmt.Errorf("The policy doesn't exist in DB, policy id:%d", job.PolicyID) + } + sm.Parms = &RepJobParm{ + LocalRegURL: regURL, + Repository: job.Repository, + Enabled: policy.Enabled, + Operation: job.Operation, + } + if policy.Enabled == 0 { + //handler will cancel this job + return nil + } + target, err := dao.GetRepTarget(policy.TargetID) + if err != nil { + return fmt.Errorf("Failed to get target, error: %v", err) + } + if target == nil { + return fmt.Errorf("The target doesn't exist in DB, target id: %d", policy.TargetID) + } + sm.Parms.TargetURL = target.URL + sm.Parms.TargetUsername = target.Username + sm.Parms.TargetPassword = target.Password + //init states handlers sm.lock = &sync.Mutex{} sm.Handlers = make(map[string]StateHandler) sm.Transitions = make(map[string]map[string]struct{}) - sm.CurrentState = JobPending - sm.AddTransition(JobPending, JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, JobRunning}) - sm.Handlers[JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, JobError} - sm.Handlers[JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, JobStopped} + sm.Logger = Logger{sm.JobID} + sm.CurrentState = models.JobPending + sm.AddTransition(models.JobPending, models.JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobRunning}) + sm.Handlers[models.JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobError} + sm.Handlers[models.JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobStopped} + if sm.Parms.Operation == models.RepOpTransfer { + sm.AddTransition(models.JobRunning, "pull-img", ImgPuller{DummyHandler: DummyHandler{JobID: sm.JobID}, img: sm.Parms.Repository, logger: sm.Logger}) + //only handle on target for now + sm.AddTransition("pull-img", "push-img", ImgPusher{DummyHandler: DummyHandler{JobID: sm.JobID}, targetURL: sm.Parms.TargetURL, logger: sm.Logger}) + sm.AddTransition("push-img", models.JobFinished, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished}) + } + return nil } diff --git a/jobservice/my_start.sh b/jobservice/my_start.sh index d2274be48..bbcff279c 100755 --- a/jobservice/my_start.sh +++ b/jobservice/my_start.sh @@ -3,5 +3,8 @@ export MYSQL_PORT=3306 export MYSQL_USR=root export MYSQL_PWD=root123 export LOG_LEVEL=debug +export UI_URL=http://127.0.0.1/ +export UI_USR=admin +export UI_PWD=Harbor12345 ./jobservice diff --git a/jobservice/newtest.json b/jobservice/newtest.json new file mode 100644 index 000000000..9a40fa461 --- /dev/null +++ b/jobservice/newtest.json @@ -0,0 +1 @@ +{"policy_id": 1} diff --git a/jobservice/populate.sql b/jobservice/populate.sql new file mode 100644 index 000000000..64fb2d360 --- /dev/null +++ b/jobservice/populate.sql @@ -0,0 +1,2 @@ +insert into replication_target (name, url, username, password) values ('test', '192.168.0.2:5000', 'testuser', 'passw0rd'); +insert into replication_policy (name, project_id, target_id, enabled, start_time) value ('test_policy', 1, 1, 1, NOW()); diff --git a/jobservice/router.go b/jobservice/router.go index eb70e52fa..a9c028038 100644 --- a/jobservice/router.go +++ b/jobservice/router.go @@ -7,5 +7,5 @@ import ( ) func initRouters() { - beego.Router("/api/jobs/?:id", &api.JobAPI{}) + beego.Router("/api/jobs/replication", &api.ReplicationJob{}) } diff --git a/migration/Dockerfile b/migration/Dockerfile new file mode 100644 index 000000000..507342170 --- /dev/null +++ b/migration/Dockerfile @@ -0,0 +1,23 @@ +FROM mysql:5.6 + +MAINTAINER bhe@vmware.com + +RUN sed -i -e 's/us.archive.ubuntu.com/archive.ubuntu.com/g' /etc/apt/sources.list + +RUN apt-get update + +RUN apt-get install -y curl python python-pip git python-mysqldb + +RUN pip install alembic + +RUN mkdir -p /harbor-migration + +WORKDIR /harbor-migration + +COPY ./ ./ + +COPY ./migration.cfg ./ + +RUN ./prepare.sh + +ENTRYPOINT ["./run.sh"] diff --git a/migration/README.md b/migration/README.md new file mode 100644 index 000000000..1d013e47f --- /dev/null +++ b/migration/README.md @@ -0,0 +1,51 @@ +# migration +Migration is a module for migrating database schema between different version of project [harbor](https://github.com/vmware/harbor) + +**WARNING!!** You must backup your data before migrating + +###installation +- step 1: modify migration.cfg +- step 2: build image from dockerfile + ``` + cd harbor-migration + + docker build -t your-image-name . + ``` + +###migration operation +- show instruction of harbor-migration + + ```docker run your-image-name help``` + +- create backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name backup + ``` + +- restore from backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name restore + ``` + +- perform database schema upgrade + + ```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head``` + +- perform database schema downgrade(downgrade has been disabled) + + ```docker run -v /data/database:/var/lib/mysql your-image-name down base``` + +###migration step +- step 1: stop and remove harbor service + + ``` + docker-compose stop && docker-compose rm -f + ``` +- step 2: perform migration operation +- step 3: rebuild newest harbor images and restart service + + ``` + docker-compose build && docker-compose up -d + ``` diff --git a/migration/alembic.sql b/migration/alembic.sql new file mode 100644 index 000000000..21dc7a1de --- /dev/null +++ b/migration/alembic.sql @@ -0,0 +1,4 @@ +use `registry`; +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/migration/alembic.tpl b/migration/alembic.tpl new file mode 100644 index 000000000..548c6028d --- /dev/null +++ b/migration/alembic.tpl @@ -0,0 +1,68 @@ +echo " +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migration_harbor + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to migration_harbor/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat migration_harbor/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = mysql://$db_username:$db_password@localhost:$db_port/$db_name + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S" diff --git a/migration/migration.cfg b/migration/migration.cfg new file mode 100644 index 000000000..a383853ac --- /dev/null +++ b/migration/migration.cfg @@ -0,0 +1,4 @@ +db_username="root" +db_password="root123" +db_port="3306" +db_name="registry" diff --git a/migration/migration_harbor/env.py b/migration/migration_harbor/env.py new file mode 100644 index 000000000..646f39862 --- /dev/null +++ b/migration/migration_harbor/env.py @@ -0,0 +1,85 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata + +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migration/migration_harbor/script.py.mako b/migration/migration_harbor/script.py.mako new file mode 100644 index 000000000..43c09401b --- /dev/null +++ b/migration/migration_harbor/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migration/migration_harbor/versions/0_1_1.py b/migration/migration_harbor/versions/0_1_1.py new file mode 100644 index 000000000..0f21b5436 --- /dev/null +++ b/migration/migration_harbor/versions/0_1_1.py @@ -0,0 +1,128 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""0.1.0 to 0.1.1 + +Revision ID: 0.1.1 +Revises: +Create Date: 2016-04-18 18:32:14.101897 + +""" + +# revision identifiers, used by Alembic. +revision = '0.1.1' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, relationship +from datetime import datetime + +Session = sessionmaker() + +Base = declarative_base() + +class Properties(Base): + __tablename__ = 'properties' + + k = sa.Column(sa.String(64), primary_key = True) + v = sa.Column(sa.String(128), nullable = False) + +class ProjectMember(Base): + __tablename__ = 'project_member' + + project_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), primary_key = True) + role = sa.Column(sa.Integer(), nullable = False) + creation_time = sa.Column(sa.DateTime(), nullable = True) + update_time = sa.Column(sa.DateTime(), nullable = True) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ), + sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ), + sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ), + +class UserProjectRole(Base): + __tablename__ = 'user_project_role' + + upr_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id')) + pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id')) + project_role = relationship("ProjectRole") + +class ProjectRole(Base): + __tablename__ = 'project_role' + + pr_id = sa.Column(sa.Integer(), primary_key = True) + project_id = sa.Column(sa.Integer(), nullable = False) + role_id = sa.Column(sa.Integer(), nullable = False) + sa.ForeignKeyConstraint(['role_id'], [u'role.role_id']) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id']) + +class Access(Base): + __tablename__ = 'access' + + access_id = sa.Column(sa.Integer(), primary_key = True) + access_code = sa.Column(sa.String(1)) + comment = sa.Column(sa.String(30)) + +def upgrade(): + """ + update schema&data + """ + bind = op.get_bind() + session = Session(bind=bind) + + #delete M from table access + acc = session.query(Access).filter_by(access_id=1).first() + session.delete(acc) + + #create table property + Properties.__table__.create(bind) + session.add(Properties(k='schema_version', v='0.1.1')) + + #create table project_member + ProjectMember.__table__.create(bind) + + #fill data + join_result = session.query(UserProjectRole).join(UserProjectRole.project_role).all() + for result in join_result: + session.add(ProjectMember(project_id=result.project_role.project_id, \ + user_id=result.user_id, role=result.project_role.role_id, \ + creation_time=datetime.now(), update_time=datetime.now())) + + #drop user_project_role table before drop project_role + #because foreign key constraint + op.drop_table('user_project_role') + op.drop_table('project_role') + + #add column to table project + op.add_column('project', sa.Column('update_time', sa.DateTime(), nullable=True)) + + #add column to table role + op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False)) + + #add column to table user + op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True)) + op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True)) + op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True)) + session.commit() + +def downgrade(): + """ + Downgrade has been disabled. + """ + pass diff --git a/migration/prepare.sh b/migration/prepare.sh new file mode 100755 index 000000000..4d707407c --- /dev/null +++ b/migration/prepare.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source ./migration.cfg +source ./alembic.tpl > ./alembic.ini diff --git a/migration/run.sh b/migration/run.sh new file mode 100755 index 000000000..806378ff5 --- /dev/null +++ b/migration/run.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +source ./migration.cfg + +WAITTIME=60 + +DBCNF="-hlocalhost -u${db_username}" + +#prevent shell to print insecure message +export MYSQL_PWD="${db_password}" + +if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then + echo "Usage:" + echo "backup perform database backup" + echo "restore perform database restore" + echo "up, upgrade perform database schema upgrade" + echo "h, help usage help" + exit 0 +fi + +if [[ $1 = "up" || $1 = "upgrade" ]]; then + echo "Please backup before upgrade." + read -p "Enter y to continue updating or n to abort:" ans + case $ans in + [Yy]* ) + ;; + [Nn]* ) + exit 0 + ;; + * ) echo "illegal answer: $ans. Upgrade abort!!" + exit 1 + ;; + esac + +fi + +echo 'Trying to start mysql server...' +DBRUN=0 +nohup mysqld 2>&1 > ./nohup.log& +for i in $(seq 1 $WAITTIME); do + echo "$(/usr/sbin/service mysql status)" + if [[ "$(/usr/sbin/service mysql status)" =~ "not running" ]]; then + sleep 1 + else + DBRUN=1 + break + fi +done + +if [[ $DBRUN -eq 0 ]]; then + echo "timeout. Can't run mysql server." + exit 1 +fi + +key="$1" +case $key in +up|upgrade) + VERSION="$2" + if [[ -z $VERSION ]]; then + VERSION="head" + echo "Version is not specified. Default version is head." + fi + echo "Performing upgrade ${VERSION}..." + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='alembic_version';") -eq 0 ]]; then + echo "table alembic_version does not exist. Trying to initial alembic_version." + mysql $DBCNF < ./alembic.sql + #compatible with version 0.1.0 and 0.1.1 + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='properties'") -eq 0 ]]; then + echo "table properties does not exist. The version of registry is 0.1.0" + else + echo "The version of registry is 0.1.1" + mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')" + fi + fi + alembic -c ./alembic.ini upgrade ${VERSION} + echo "Upgrade performed." + ;; +backup) + echo "Performing backup..." + mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql + echo "Backup performed." + ;; +restore) + echo "Performing restore..." + mysql $DBCNF < ./backup/registry.sql + echo "Restore performed." + ;; +*) + echo "unknown option" + exit 0 + ;; +esac diff --git a/models/base.go b/models/base.go new file mode 100644 index 000000000..fdd0204ac --- /dev/null +++ b/models/base.go @@ -0,0 +1,11 @@ +package models + +import ( + "github.com/astaxie/beego/orm" +) + +func init() { + orm.RegisterModel(new(RepTarget), + new(RepPolicy), + new(RepJob)) +} diff --git a/models/replication_job.go b/models/replication_job.go new file mode 100644 index 000000000..885cb311c --- /dev/null +++ b/models/replication_job.go @@ -0,0 +1,65 @@ +package models + +import ( + "time" +) + +const ( + JobPending string = "pending" + JobRunning string = "running" + JobError string = "error" + JobStopped string = "stopped" + JobFinished string = "finished" + JobCanceled string = "canceled" + // statemachine will move to next possible state based on trasition table + JobContinue string = "_continue" + RepOpTransfer string = "transfer" + RepOpDelete string = "delete" +) + +type RepPolicy struct { + ID int64 `orm:"column(id)" json:"id"` + ProjectID int64 `orm:"column(project_id)" json:"project_id"` + TargetID int64 `orm:"column(target_id)" json:"target_id"` + Name string `orm:"column(name)" json:"name"` + Target RepTarget `orm:"-" json:"target"` + Enabled int `orm:"column(enabled)" json:"enabled"` + Description string `orm:"column(description)" json:"description"` + CronStr string `orm:"column(cron_str)" json:"cron_str"` + StartTime time.Time `orm:"column(start_time)" json:"start_time"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +type RepJob struct { + ID int64 `orm:"column(id)" json:"id"` + Status string `orm:"column(status)" json:"status"` + Repository string `orm:"column(repository)" json:"repository"` + PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` + Operation string `orm:"column(operation)" json:"operation"` + Policy RepPolicy `orm:"-" json:"policy"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +type RepTarget struct { + ID int64 `orm:"column(id)" json:"id"` + URL string `orm:"column(url)" json:"url"` + Name string `orm:"column(name)" json:"name"` + Username string `orm:"column(username)" json:"username"` + Password string `orm:"column(password)" json:"password"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +func (rt *RepTarget) TableName() string { + return "replication_target" +} + +func (rj *RepJob) TableName() string { + return "replication_job" +} + +func (rp *RepPolicy) TableName() string { + return "replication_policy" +} diff --git a/utils/registry/auth/tokenhandler.go b/utils/registry/auth/tokenhandler.go index c33627491..f546bac0c 100644 --- a/utils/registry/auth/tokenhandler.go +++ b/utils/registry/auth/tokenhandler.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "net/http" "net/url" - "strconv" "strings" "time" @@ -176,7 +175,7 @@ func (s *standardTokenHandler) generateToken(realm, service string, scopes []str tk := struct { Token string `json:"token"` - ExpiresIn string `json:"expires_in"` + ExpiresIn int `json:"expires_in"` IssuedAt string `json:"issued_at"` }{} if err = json.Unmarshal(b, &tk); err != nil { @@ -185,19 +184,14 @@ func (s *standardTokenHandler) generateToken(realm, service string, scopes []str token = tk.Token - expiresIn, err = strconv.Atoi(tk.ExpiresIn) + expiresIn = tk.ExpiresIn + + t, err := time.Parse(time.RFC3339, tk.IssuedAt) if err != nil { - expiresIn = 0 - log.Errorf("error occurred while converting expires_in: %v", err) + log.Errorf("error occurred while parsing issued_at: %v", err) err = nil } else { - t, err := time.Parse(time.RFC3339, tk.IssuedAt) - if err != nil { - log.Errorf("error occurred while parsing issued_at: %v", err) - err = nil - } else { - issuedAt = &t - } + issuedAt = &t } log.Debug("get token from token server") diff --git a/utils/registry/repository.go b/utils/registry/repository.go index 0a1b9f096..507634415 100644 --- a/utils/registry/repository.go +++ b/utils/registry/repository.go @@ -103,7 +103,7 @@ func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, er client: client, } - log.Debugf("initialized a repository client with username: %s %s", endpoint, name, username) + log.Debugf("initialized a repository client with username: %s %s %s", endpoint, name, username) return repository, nil } diff --git a/views/segment/header-content.tpl b/views/segment/header-content.tpl index 464cd09aa..368a64d8c 100644 --- a/views/segment/header-content.tpl +++ b/views/segment/header-content.tpl @@ -57,6 +57,10 @@
  •   {{i18n .Lang "change_password"}}
    • {{ end }} + {{ if eq .IsLdapAdminUser true }} +
  •   {{i18n .Lang "change_password"}}
    • + + {{ end }} {{ if eq .AuthMode "db_auth" }} {{ if eq .IsAdmin true }}
  •   {{i18n .Lang "add_user"}}