From 9493b8c238006a346b1a077254c1d59db820be2d Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Fri, 27 May 2016 16:21:32 +0800 Subject: [PATCH] integrate jobservice into docker-compose template --- .gitignore | 2 +- Deploy/docker-compose.yml | 13 +++++++++++++ Deploy/harbor.cfg | 5 ++++- Deploy/prepare | 23 ++++++++++++++++++++--- Deploy/templates/jobservice/env | 9 +++++++++ Deploy/templates/ui/env | 1 + Dockerfile.job | 19 +++++++++++++++++++ api/jobs/replication.go | 2 +- job/config/config.go | 4 ++-- jobservice/conf/app.conf | 5 +++++ 10 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 Deploy/templates/jobservice/env create mode 100644 Dockerfile.job create mode 100644 jobservice/conf/app.conf diff --git a/.gitignore b/.gitignore index b53274b48..906c9c7cd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ Deploy/config/registry/config.yml Deploy/config/ui/env Deploy/config/ui/app.conf Deploy/config/db/env -Deploy/harbor.cfg +Deploy/config/jobservice/env ui/ui *.pyc diff --git a/Deploy/docker-compose.yml b/Deploy/docker-compose.yml index 2dc09b620..a57475ef1 100644 --- a/Deploy/docker-compose.yml +++ b/Deploy/docker-compose.yml @@ -53,6 +53,19 @@ services: options: syslog-address: "tcp://127.0.0.1:1514" syslog-tag: "ui" + jobservice: + build: + context: ../ + dockerfile: Dockerfile.job + env_file: + - ./config/jobservice/env + depends_on: + - ui + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + syslog-tag: "jobservice" proxy: image: library/nginx:1.9 volumes: diff --git a/Deploy/harbor.cfg b/Deploy/harbor.cfg index bd949cea5..e854d289c 100644 --- a/Deploy/harbor.cfg +++ b/Deploy/harbor.cfg @@ -2,7 +2,7 @@ #The IP address or hostname to access admin UI and registry service. #DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. -hostname = reg.mydomain.com +hostname = reg.mydomain.org #The protocol for accessing the UI and token/notification service, by default it is http. #It can be set to https if ssl is enabled on nginx. @@ -38,6 +38,9 @@ self_registration = on #Turn on or off the customize your certicate customize_crt = on +#Number of job workers in job service, default is 10 +max_job_workers = 10 + #fill in your certicate message crt_country = CN crt_state = State diff --git a/Deploy/prepare b/Deploy/prepare index edd31ea95..d0afc365c 100755 --- a/Deploy/prepare +++ b/Deploy/prepare @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- from __future__ import print_function, unicode_literals # We require Python 2.6 or later from string import Template +import random +import string import os import sys from io import open @@ -44,13 +46,15 @@ crt_organization = rcp.get("configuration", "crt_organization") crt_organizationalunit = rcp.get("configuration", "crt_organizationalunit") crt_commonname = rcp.get("configuration", "crt_commonname") crt_email = rcp.get("configuration", "crt_email") +max_job_workers = rcp.get("configuration", "max_job_workers") ######## +ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) + base_dir = os.path.dirname(__file__) config_dir = os.path.join(base_dir, "config") templates_dir = os.path.join(base_dir, "templates") - ui_config_dir = os.path.join(config_dir,"ui") if not os.path.exists(ui_config_dir): os.makedirs(os.path.join(config_dir, "ui")) @@ -59,6 +63,10 @@ db_config_dir = os.path.join(config_dir, "db") if not os.path.exists(db_config_dir): os.makedirs(os.path.join(config_dir, "db")) +job_config_dir = os.path.join(config_dir, "jobservice") +if not os.path.exists(job_config_dir): + os.makedirs(job_config_dir) + def render(src, dest, **kw): t = Template(open(src, 'r').read()) with open(dest, 'w') as f: @@ -69,8 +77,9 @@ ui_conf_env = os.path.join(config_dir, "ui", "env") ui_conf = os.path.join(config_dir, "ui", "app.conf") registry_conf = os.path.join(config_dir, "registry", "config.yml") db_conf_env = os.path.join(config_dir, "db", "env") +job_conf_env = os.path.join(config_dir, "jobservice", "env") -conf_files = [ ui_conf, ui_conf_env, registry_conf, db_conf_env ] +conf_files = [ ui_conf, ui_conf_env, registry_conf, db_conf_env, job_conf_env ] def rmdir(cf): for f in cf: if os.path.exists(f): @@ -87,7 +96,8 @@ render(os.path.join(templates_dir, "ui", "env"), harbor_admin_password=harbor_admin_password, ldap_url=ldap_url, ldap_basedn=ldap_basedn, - self_registration=self_registration) + self_registration=self_registration, + ui_secret=ui_secret) render(os.path.join(templates_dir, "ui", "app.conf"), ui_conf, @@ -107,6 +117,13 @@ render(os.path.join(templates_dir, "db", "env"), db_conf_env, db_password=db_password) +render(os.path.join(templates_dir, "jobservice", "env"), + job_conf_env, + db_password=db_password, + ui_secret=ui_secret, + max_job_workers=max_job_workers, + ui_url=ui_url) + def validate_crt_subj(dirty_subj): subj_list = [item for item in dirty_subj.strip().split("/") \ if len(item.split("=")) == 2 and len(item.split("=")[1]) > 0] diff --git a/Deploy/templates/jobservice/env b/Deploy/templates/jobservice/env new file mode 100644 index 000000000..f40d836f1 --- /dev/null +++ b/Deploy/templates/jobservice/env @@ -0,0 +1,9 @@ +MYSQL_HOST=mysql +MYSQL_PORT=3306 +MYSQL_USR=root +MYSQL_PWD=$db_password +UI_SECRET=$ui_secret +HARBOR_URL=$ui_url +MAX_JOB_WORKERS=10 +LOG_LEVEL=debug +GODEBUG=netdns=cgo diff --git a/Deploy/templates/ui/env b/Deploy/templates/ui/env index d21e8fafc..de50c4a76 100644 --- a/Deploy/templates/ui/env +++ b/Deploy/templates/ui/env @@ -10,6 +10,7 @@ HARBOR_URL=$hostname AUTH_MODE=$auth_mode LDAP_URL=$ldap_url LDAP_BASE_DN=$ldap_basedn +UI_SECRET=$ui_secret SELF_REGISTRATION=$self_registration LOG_LEVEL=debug GODEBUG=netdns=cgo diff --git a/Dockerfile.job b/Dockerfile.job new file mode 100644 index 000000000..4315d71a5 --- /dev/null +++ b/Dockerfile.job @@ -0,0 +1,19 @@ +FROM golang:1.6.2 + +MAINTAINER jiangd@vmware.com + +RUN apt-get update \ + && apt-get install -y libldap2-dev \ + && rm -r /var/lib/apt/lists/* +COPY . /go/src/github.com/vmware/harbor + +WORKDIR /go/src/github.com/vmware/harbor/jobservice + +RUN go get -d github.com/docker/distribution \ + && go get -d github.com/docker/libtrust \ + && go get -d github.com/go-sql-driver/mysql \ + && go build -v -a -o /go/bin/harbor_jobservice \ + && chmod u+x /go/bin/harbor_jobservice +ADD ./jobservice/conf /go/bin/conf +WORKDIR /go/bin/ +ENTRYPOINT ["/go/bin/harbor_jobservice"] diff --git a/api/jobs/replication.go b/api/jobs/replication.go index 7a457f057..7e847df75 100644 --- a/api/jobs/replication.go +++ b/api/jobs/replication.go @@ -144,7 +144,7 @@ func getRepoList(projectID int64) ([]string, error) { */ uiURL := config.LocalHarborURL() client := &http.Client{} - req, err := http.NewRequest("GET", uiURL+"api/repositories?project_id="+strconv.Itoa(int(projectID)), nil) + req, err := http.NewRequest("GET", uiURL+"/api/repositories?project_id="+strconv.Itoa(int(projectID)), nil) if err != nil { log.Errorf("Error when creating request: %v") return nil, err diff --git a/job/config/config.go b/job/config/config.go index 95a16fc7e..770d3625e 100644 --- a/job/config/config.go +++ b/job/config/config.go @@ -24,9 +24,9 @@ func init() { maxJobWorkers = defaultMaxWorkers } - localURL = os.Getenv("LOCAL_HARBOR_URL") + localURL = os.Getenv("HARBOR_URL") if len(localURL) == 0 { - localURL = "http://ui/" + localURL = "http://registry:5000/" } logDir = os.Getenv("LOG_DIR") diff --git a/jobservice/conf/app.conf b/jobservice/conf/app.conf new file mode 100644 index 000000000..21439a8e1 --- /dev/null +++ b/jobservice/conf/app.conf @@ -0,0 +1,5 @@ +appname = jobservice +runmode = dev + +[dev] +httpport = 80