Merge pull request #279 from reasonerjt/job-service

Job service integration

.gitignore

@@ -3,6 +3,6 @@ Deploy/config/registry/config.yml
 Deploy/config/ui/env
 Deploy/config/ui/app.conf
 Deploy/config/db/env
-Deploy/harbor.cfg
+Deploy/config/jobservice/env
 ui/ui
 *.pyc

Deploy/docker-compose.yml

@@ -53,6 +53,19 @@ services:
       options:  
         syslog-address: "tcp://127.0.0.1:1514"
         syslog-tag: "ui"
+  jobservice:
+    build:
+      context: ../
+      dockerfile: Dockerfile.job
+    env_file:
+      - ./config/jobservice/env
+    depends_on:
+      - ui
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        syslog-tag: "jobservice"
   proxy:
     image: library/nginx:1.9
     volumes:

Deploy/harbor.cfg

@@ -2,7 +2,7 @@
 
 #The IP address or hostname to access admin UI and registry service.
 #DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
-hostname = reg.mydomain.com
+hostname = reg.mydomain.org
 
 #The protocol for accessing the UI and token/notification service, by default it is http.
 #It can be set to https if ssl is enabled on nginx.
@@ -38,6 +38,9 @@ self_registration = on
 #Turn on or off the customize your certicate
 customize_crt = on
 
+#Number of job workers in job service, default is 10
+max_job_workers = 10
+
 #fill in your certicate message
 crt_country = CN
 crt_state = State

Deploy/prepare

@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import print_function, unicode_literals # We require Python 2.6 or later
 from string import Template
+import random
+import string
 import os
 import sys
 from io import open
@@ -44,13 +46,15 @@ crt_organization = rcp.get("configuration", "crt_organization")
 crt_organizationalunit = rcp.get("configuration", "crt_organizationalunit")
 crt_commonname = rcp.get("configuration", "crt_commonname")
 crt_email = rcp.get("configuration", "crt_email")
+max_job_workers = rcp.get("configuration", "max_job_workers")
 ########
 
+ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))  
+
 base_dir = os.path.dirname(__file__)
 config_dir = os.path.join(base_dir, "config")
 templates_dir = os.path.join(base_dir, "templates")
 
-
 ui_config_dir = os.path.join(config_dir,"ui")
 if not os.path.exists(ui_config_dir):
     os.makedirs(os.path.join(config_dir, "ui"))
@@ -59,6 +63,10 @@ db_config_dir = os.path.join(config_dir, "db")
 if not os.path.exists(db_config_dir):
     os.makedirs(os.path.join(config_dir, "db"))
 
+job_config_dir = os.path.join(config_dir, "jobservice")
+if not os.path.exists(job_config_dir):
+    os.makedirs(job_config_dir)
+
 def render(src, dest, **kw):
     t = Template(open(src, 'r').read())
     with open(dest, 'w') as f:
@@ -69,8 +77,9 @@ ui_conf_env = os.path.join(config_dir, "ui", "env")
 ui_conf = os.path.join(config_dir, "ui", "app.conf")
 registry_conf = os.path.join(config_dir, "registry", "config.yml")
 db_conf_env = os.path.join(config_dir, "db", "env")
+job_conf_env = os.path.join(config_dir, "jobservice", "env")
 
-conf_files = [ ui_conf, ui_conf_env, registry_conf, db_conf_env ]
+conf_files = [ ui_conf, ui_conf_env, registry_conf, db_conf_env, job_conf_env ]
 def rmdir(cf):
     for f in cf:
         if os.path.exists(f):
@@ -87,7 +96,8 @@ render(os.path.join(templates_dir, "ui", "env"),
         harbor_admin_password=harbor_admin_password,
         ldap_url=ldap_url,
         ldap_basedn=ldap_basedn,
-	self_registration=self_registration)
+	self_registration=self_registration,
+        ui_secret=ui_secret)
 
 render(os.path.join(templates_dir, "ui", "app.conf"),
         ui_conf,
@@ -107,6 +117,13 @@ render(os.path.join(templates_dir, "db", "env"),
         db_conf_env,
         db_password=db_password)
 
+render(os.path.join(templates_dir, "jobservice", "env"),
+        job_conf_env,
+        db_password=db_password,
+        ui_secret=ui_secret,
+        max_job_workers=max_job_workers,
+        ui_url=ui_url)
+
 def validate_crt_subj(dirty_subj):
     subj_list = [item for item in dirty_subj.strip().split("/") \
         if len(item.split("=")) == 2 and len(item.split("=")[1]) > 0]

Deploy/templates/jobservice/env

@@ -0,0 +1,9 @@
+MYSQL_HOST=mysql
+MYSQL_PORT=3306
+MYSQL_USR=root
+MYSQL_PWD=$db_password
+UI_SECRET=$ui_secret
+HARBOR_URL=$ui_url
+MAX_JOB_WORKERS=10
+LOG_LEVEL=debug
+GODEBUG=netdns=cgo

Deploy/templates/ui/env

@@ -10,6 +10,7 @@ HARBOR_URL=$hostname
 AUTH_MODE=$auth_mode
 LDAP_URL=$ldap_url
 LDAP_BASE_DN=$ldap_basedn
+UI_SECRET=$ui_secret
 SELF_REGISTRATION=$self_registration
 LOG_LEVEL=debug
 GODEBUG=netdns=cgo

Dockerfile.job

@@ -0,0 +1,19 @@
+FROM golang:1.6.2
+
+MAINTAINER jiangd@vmware.com
+
+RUN apt-get update \
+    && apt-get install -y libldap2-dev \
+    && rm -r /var/lib/apt/lists/*
+COPY . /go/src/github.com/vmware/harbor
+
+WORKDIR /go/src/github.com/vmware/harbor/jobservice
+
+RUN go get -d github.com/docker/distribution \
+    && go get -d github.com/docker/libtrust \
+    && go get -d github.com/go-sql-driver/mysql \
+    && go build -v -a -o /go/bin/harbor_jobservice \
+    && chmod u+x /go/bin/harbor_jobservice 
+ADD ./jobservice/conf /go/bin/conf
+WORKDIR /go/bin/
+ENTRYPOINT ["/go/bin/harbor_jobservice"]

api/jobs/replication.go

@@ -144,7 +144,7 @@ func getRepoList(projectID int64) ([]string, error) {
 	*/
 	uiURL := config.LocalHarborURL()
 	client := &http.Client{}
-	req, err := http.NewRequest("GET", uiURL+"api/repositories?project_id="+strconv.Itoa(int(projectID)), nil)
+	req, err := http.NewRequest("GET", uiURL+"/api/repositories?project_id="+strconv.Itoa(int(projectID)), nil)
 	if err != nil {
 		log.Errorf("Error when creating request: %v")
 		return nil, err

job/config/config.go

@@ -24,9 +24,9 @@ func init() {
 		maxJobWorkers = defaultMaxWorkers
 	}
 
-	localURL = os.Getenv("LOCAL_HARBOR_URL")
+	localURL = os.Getenv("HARBOR_URL")
 	if len(localURL) == 0 {
-		localURL = "http://ui/"
+		localURL = "http://registry:5000/"
 	}
 
 	logDir = os.Getenv("LOG_DIR")

jobservice/conf/app.conf

@@ -0,0 +1,5 @@
+appname = jobservice
+runmode = dev
+
+[dev]
+httpport = 80