Improve the performance of database when paginating large records for access logs, jobs and getting the top repositories

Deploy/db/registry.sql

@@ -107,6 +107,7 @@ create table access_log (
  operation varchar(20) NOT NULL,
  op_time timestamp,
  primary key (log_id),
+ INDEX pid_optime (project_id, op_time),
  FOREIGN KEY (user_id) REFERENCES user(user_id),
  FOREIGN KEY (project_id) REFERENCES project (project_id)
 );
@@ -169,7 +170,8 @@ create table replication_job (
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
  PRIMARY KEY (id),
- INDEX policy (policy_id)
+ INDEX policy (policy_id),
+ INDEX poid_uptime (policy_id, update_time)
  );
  
 create table properties (

api/project.go

@@ -339,13 +339,18 @@ func (p *ProjectAPI) FilterAccessLog() {
 	p.DecodeJSONReq(&query)
 
 	query.ProjectID = p.projectID
-	query.Username = "%" + query.Username + "%"
 	query.BeginTime = time.Unix(query.BeginTimestamp, 0)
 	query.EndTime = time.Unix(query.EndTimestamp, 0)
 
 	page, pageSize := p.getPaginationParams()
 
-	logs, total, err := dao.GetAccessLogs(query, pageSize, pageSize*(page-1))
+	total, err := dao.GetTotalOfAccessLogs(query)
+	if err != nil {
+		log.Errorf("failed to get total of access log: %v", err)
+		p.CustomAbort(http.StatusInternalServerError, "")
+	}
+
+	logs, err := dao.GetAccessLogs(query, pageSize, pageSize*(page-1))
 	if err != nil {
 		log.Errorf("failed to get access log: %v", err)
 		p.CustomAbort(http.StatusInternalServerError, "")

dao/accesslog.go

@@ -38,79 +38,106 @@ func AddAccessLog(accessLog models.AccessLog) error {
 	return err
 }
 
-//GetAccessLogs gets access logs according to different conditions
+// GetTotalOfAccessLogs ...
-func GetAccessLogs(query models.AccessLog, limit, offset int64) ([]models.AccessLog, int64, error) {
+func GetTotalOfAccessLogs(query models.AccessLog) (int64, error) {
 	o := GetOrmer()
 
-	condition := ` from access_log a left join user u on a.user_id = u.user_id
+	queryParam := []interface{}{}
-		where a.project_id = ? `
+
-	queryParam := make([]interface{}, 1)
+	sql := `select count(*) from access_log al 
+		where al.project_id = ?`
 	queryParam = append(queryParam, query.ProjectID)
 
-	if query.UserID != 0 {
-		condition += ` and a.user_id = ? `
-		queryParam = append(queryParam, query.UserID)
-	}
-	if query.Operation != "" {
-		condition += ` and a.operation = ? `
-		queryParam = append(queryParam, query.Operation)
-	}
 	if query.Username != "" {
-		condition += ` and u.username like ? `
+		sql = `select count(*) from access_log al 
-		queryParam = append(queryParam, query.Username)
+			left join user u 
+			on al.user_id = u.user_id 
+			where al.project_id = ? and u.username like ? `
+		queryParam = append(queryParam, "%"+query.Username+"%")
+	}
+
+	sql += genFilterClauses(query, &queryParam)
+
+	var total int64
+	if err := o.Raw(sql, queryParam).QueryRow(&total); err != nil {
+		return 0, err
+	}
+	return total, nil
+}
+
+//GetAccessLogs gets access logs according to different conditions
+func GetAccessLogs(query models.AccessLog, limit, offset int64) ([]models.AccessLog, error) {
+	o := GetOrmer()
+
+	queryParam := []interface{}{}
+	sql := `select al.log_id, u.username, al.repo_name, 
+			al.repo_tag, al.operation, al.op_time 
+		from access_log al 
+		left join user u 
+		on al.user_id = u.user_id
+		where al.project_id = ? `
+	queryParam = append(queryParam, query.ProjectID)
+
+	if query.Username != "" {
+		sql += ` and u.username like ? `
+		queryParam = append(queryParam, "%"+query.Username+"%")
+	}
+
+	sql += genFilterClauses(query, &queryParam)
+
+	sql += ` order by al.op_time desc `
+
+	sql = paginateForRawSQL(sql, limit, offset)
+
+	logs := []models.AccessLog{}
+	_, err := o.Raw(sql, queryParam).QueryRows(&logs)
+	if err != nil {
+		return logs, err
+	}
+
+	return logs, nil
+}
+
+func genFilterClauses(query models.AccessLog, queryParam *[]interface{}) string {
+	sql := ""
+
+	if query.Operation != "" {
+		sql += ` and al.operation = ? `
+		*queryParam = append(*queryParam, query.Operation)
 	}
 	if query.RepoName != "" {
-		condition += ` and a.repo_name = ? `
+		sql += ` and al.repo_name = ? `
-		queryParam = append(queryParam, query.RepoName)
+		*queryParam = append(*queryParam, query.RepoName)
 	}
 	if query.RepoTag != "" {
-		condition += ` and a.repo_tag = ? `
+		sql += ` and al.repo_tag = ? `
-		queryParam = append(queryParam, query.RepoTag)
+		*queryParam = append(*queryParam, query.RepoTag)
 	}
 	if query.Keywords != "" {
-		condition += ` and a.operation in ( `
+		sql += ` and al.operation in ( `
 		keywordList := strings.Split(query.Keywords, "/")
 		num := len(keywordList)
 		for i := 0; i < num; i++ {
 			if keywordList[i] != "" {
 				if i == num-1 {
-					condition += `?)`
+					sql += `?)`
 				} else {
-					condition += `?,`
+					sql += `?,`
 				}
-				queryParam = append(queryParam, keywordList[i])
+				*queryParam = append(*queryParam, keywordList[i])
 			}
 		}
 	}
 	if query.BeginTimestamp > 0 {
-		condition += ` and a.op_time >= ? `
+		sql += ` and al.op_time >= ? `
-		queryParam = append(queryParam, query.BeginTime)
+		*queryParam = append(*queryParam, query.BeginTime)
 	}
 	if query.EndTimestamp > 0 {
-		condition += ` and a.op_time <= ? `
+		sql += ` and al.op_time <= ? `
-		queryParam = append(queryParam, query.EndTime)
+		*queryParam = append(*queryParam, query.EndTime)
 	}
 
-	condition += ` order by a.op_time desc `
+	return sql
-
-	totalSQL := `select count(*) ` + condition
-
-	logs := []models.AccessLog{}
-
-	var total int64
-	if err := o.Raw(totalSQL, queryParam).QueryRow(&total); err != nil {
-		return logs, 0, err
-	}
-
-	condition = paginateForRawSQL(condition, limit, offset)
-
-	recordsSQL := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time ` + condition
-	_, err := o.Raw(recordsSQL, queryParam).QueryRows(&logs)
-	if err != nil {
-		return logs, 0, err
-	}
-
-	return logs, total, nil
 }
 
 // AccessLog ...

dao/dao_test.go

@@ -458,7 +458,7 @@ func TestGetAccessLog(t *testing.T) {
 		UserID:    currentUser.UserID,
 		ProjectID: currentProject.ProjectID,
 	}
-	accessLogs, _, err := GetAccessLogs(queryAccessLog, 1000, 0)
+	accessLogs, err := GetAccessLogs(queryAccessLog, 1000, 0)
 	if err != nil {
 		t.Errorf("Error occurred in GetAccessLog: %v", err)
 	}
@@ -470,6 +470,21 @@ func TestGetAccessLog(t *testing.T) {
 	}
 }
 
+func TestGetTotalOfAccessLogs(t *testing.T) {
+	queryAccessLog := models.AccessLog{
+		UserID:    currentUser.UserID,
+		ProjectID: currentProject.ProjectID,
+	}
+	total, err := GetTotalOfAccessLogs(queryAccessLog)
+	if err != nil {
+		t.Fatalf("failed to get total of access log: %v", err)
+	}
+
+	if total != 1 {
+		t.Errorf("unexpected total %d != %d", total, 1)
+	}
+}
+
 func TestAddAccessLog(t *testing.T) {
 	var err error
 	var accessLogList []models.AccessLog
@@ -486,7 +501,7 @@ func TestAddAccessLog(t *testing.T) {
 	if err != nil {
 		t.Errorf("Error occurred in AddAccessLog: %v", err)
 	}
-	accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
+	accessLogList, err = GetAccessLogs(accessLog, 1000, 0)
 	if err != nil {
 		t.Errorf("Error occurred in GetAccessLog: %v", err)
 	}
@@ -515,7 +530,7 @@ func TestAccessLog(t *testing.T) {
 	if err != nil {
 		t.Errorf("Error occurred in AccessLog: %v", err)
 	}
-	accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
+	accessLogList, err = GetAccessLogs(accessLog, 1000, 0)
 	if err != nil {
 		t.Errorf("Error occurred in GetAccessLog: %v", err)
 	}