Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-30 06:03:45 +01:00
Code Issues Projects Releases Wiki Activity

Improve the performance of database when paginating large records for access logs, jobs and getting the top repositories

Browse Source
This commit is contained in:
Wenkai Yin 2016-09-02 18:34:35 +08:00
parent c3ab2e3dd4
commit bcdec42346
4 changed files with 102 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Deploy/db/registry.sql
View File

@ -107,6 +107,7 @@ create table access_log (
operation varchar(20) NOT NULL,
op_time timestamp,
primary key (log_id),
INDEX pid_optime (project_id, op_time),
FOREIGN KEY (user_id) REFERENCES user(user_id),
FOREIGN KEY (project_id) REFERENCES project (project_id)
);
@ -169,7 +170,8 @@ create table replication_job (
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
PRIMARY KEY (id),
INDEX policy (policy_id)
INDEX policy (policy_id),
INDEX poid_uptime (policy_id, update_time)
);
create table properties (

9
api/project.go
View File

@ -339,13 +339,18 @@ func (p *ProjectAPI) FilterAccessLog() {
p.DecodeJSONReq(&query)
query.ProjectID = p.projectID
query.Username = "%" + query.Username + "%"
query.BeginTime = time.Unix(query.BeginTimestamp, 0)
query.EndTime = time.Unix(query.EndTimestamp, 0)
page, pageSize := p.getPaginationParams()
logs, total, err := dao.GetAccessLogs(query, pageSize, pageSize*(page-1))
total, err := dao.GetTotalOfAccessLogs(query)
if err != nil {
log.Errorf("failed to get total of access log: %v", err)
p.CustomAbort(http.StatusInternalServerError, "")
}
logs, err := dao.GetAccessLogs(query, pageSize, pageSize*(page-1))
if err != nil {
log.Errorf("failed to get access log: %v", err)
p.CustomAbort(http.StatusInternalServerError, "")

121
dao/accesslog.go
View File

@ -38,79 +38,106 @@ func AddAccessLog(accessLog models.AccessLog) error {
return err
}
//GetAccessLogs gets access logs according to different conditions
func GetAccessLogs(query models.AccessLog, limit, offset int64) ([]models.AccessLog, int64, error) {
// GetTotalOfAccessLogs ...
func GetTotalOfAccessLogs(query models.AccessLog) (int64, error) {
o := GetOrmer()
condition := ` from access_log a left join user u on a.user_id = u.user_id
where a.project_id = ? `
queryParam := make([]interface{}, 1)
queryParam := []interface{}{}
sql := `select count(*) from access_log al
where al.project_id = ?`
queryParam = append(queryParam, query.ProjectID)
if query.UserID != 0 {
condition += ` and a.user_id = ? `
queryParam = append(queryParam, query.UserID)
}
if query.Operation != "" {
condition += ` and a.operation = ? `
queryParam = append(queryParam, query.Operation)
}
if query.Username != "" {
condition += ` and u.username like ? `
queryParam = append(queryParam, query.Username)
sql = `select count(*) from access_log al
left join user u
on al.user_id = u.user_id
where al.project_id = ? and u.username like ? `
queryParam = append(queryParam, "%"+query.Username+"%")
}
sql += genFilterClauses(query, &queryParam)
var total int64
if err := o.Raw(sql, queryParam).QueryRow(&total); err != nil {
return 0, err
}
return total, nil
}
//GetAccessLogs gets access logs according to different conditions
func GetAccessLogs(query models.AccessLog, limit, offset int64) ([]models.AccessLog, error) {
o := GetOrmer()
queryParam := []interface{}{}
sql := `select al.log_id, u.username, al.repo_name,
al.repo_tag, al.operation, al.op_time
from access_log al
left join user u
on al.user_id = u.user_id
where al.project_id = ? `
queryParam = append(queryParam, query.ProjectID)
if query.Username != "" {
sql += ` and u.username like ? `
queryParam = append(queryParam, "%"+query.Username+"%")
}
sql += genFilterClauses(query, &queryParam)
sql += ` order by al.op_time desc `
sql = paginateForRawSQL(sql, limit, offset)
logs := []models.AccessLog{}
_, err := o.Raw(sql, queryParam).QueryRows(&logs)
if err != nil {
return logs, err
}
return logs, nil
}
func genFilterClauses(query models.AccessLog, queryParam *[]interface{}) string {
sql := ""
if query.Operation != "" {
sql += ` and al.operation = ? `
*queryParam = append(*queryParam, query.Operation)
}
if query.RepoName != "" {
condition += ` and a.repo_name = ? `
queryParam = append(queryParam, query.RepoName)
sql += ` and al.repo_name = ? `
*queryParam = append(*queryParam, query.RepoName)
}
if query.RepoTag != "" {
condition += ` and a.repo_tag = ? `
queryParam = append(queryParam, query.RepoTag)
sql += ` and al.repo_tag = ? `
*queryParam = append(*queryParam, query.RepoTag)
}
if query.Keywords != "" {
condition += ` and a.operation in ( `
sql += ` and al.operation in ( `
keywordList := strings.Split(query.Keywords, "/")
num := len(keywordList)
for i := 0; i < num; i++ {
if keywordList[i] != "" {
if i == num-1 {
condition += `?)`
sql += `?)`
} else {
condition += `?,`
sql += `?,`
}
queryParam = append(queryParam, keywordList[i])
*queryParam = append(*queryParam, keywordList[i])
}
}
}
if query.BeginTimestamp > 0 {
condition += ` and a.op_time >= ? `
queryParam = append(queryParam, query.BeginTime)
sql += ` and al.op_time >= ? `
*queryParam = append(*queryParam, query.BeginTime)
}
if query.EndTimestamp > 0 {
condition += ` and a.op_time <= ? `
queryParam = append(queryParam, query.EndTime)
sql += ` and al.op_time <= ? `
*queryParam = append(*queryParam, query.EndTime)
}
condition += ` order by a.op_time desc `
totalSQL := `select count(*) ` + condition
logs := []models.AccessLog{}
var total int64
if err := o.Raw(totalSQL, queryParam).QueryRow(&total); err != nil {
return logs, 0, err
}
condition = paginateForRawSQL(condition, limit, offset)
recordsSQL := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time ` + condition
_, err := o.Raw(recordsSQL, queryParam).QueryRows(&logs)
if err != nil {
return logs, 0, err
}
return logs, total, nil
return sql
}
// AccessLog ...

21
dao/dao_test.go
View File

@ -458,7 +458,7 @@ func TestGetAccessLog(t *testing.T) {
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
}
accessLogs, _, err := GetAccessLogs(queryAccessLog, 1000, 0)
accessLogs, err := GetAccessLogs(queryAccessLog, 1000, 0)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
@ -470,6 +470,21 @@ func TestGetAccessLog(t *testing.T) {
}
}
func TestGetTotalOfAccessLogs(t *testing.T) {
queryAccessLog := models.AccessLog{
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
}
total, err := GetTotalOfAccessLogs(queryAccessLog)
if err != nil {
t.Fatalf("failed to get total of access log: %v", err)
}
if total != 1 {
t.Errorf("unexpected total %d != %d", total, 1)
}
}
func TestAddAccessLog(t *testing.T) {
var err error
var accessLogList []models.AccessLog
@ -486,7 +501,7 @@ func TestAddAccessLog(t *testing.T) {
if err != nil {
t.Errorf("Error occurred in AddAccessLog: %v", err)
}
accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
accessLogList, err = GetAccessLogs(accessLog, 1000, 0)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
@ -515,7 +530,7 @@ func TestAccessLog(t *testing.T) {
if err != nil {
t.Errorf("Error occurred in AccessLog: %v", err)
}
accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
accessLogList, err = GetAccessLogs(accessLog, 1000, 0)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}