Merge pull request #178 from tstromberg/cached-convo

optimization: use cached conversations instead of re-parsing
google · Jul 11, 2020 · 69ef214 · 69ef214
2 parents 817ea8f + 217167f
commit 69ef214
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 5 deletions.
diff --git a/pkg/hubbub/hubbub.go b/pkg/hubbub/hubbub.go
@@ -69,7 +69,7 @@ type Engine struct {
 	// Workaround because GitHub doesn't update issues if cross-references occur
 	updatedAt map[string]time.Time
 
-	// indexes used for similarity matching
+	// indexes used for similarity matching & conversation caching
 	seen map[string]*Conversation
 }
 

diff --git a/pkg/hubbub/item.go b/pkg/hubbub/item.go
@@ -62,8 +62,33 @@ type GitHubItem interface {
 	String() string
 }
 
-// conversation creates a conversation from an issue-like
+// conversation returns a cached conversation from an issue-like
 func (h *Engine) conversation(i GitHubItem, cs []*Comment, age time.Time) *Conversation {
+	key := i.GetHTMLURL()
+	cached, ok := h.seen[key]
+	if ok {
+		if cached.Updated != i.GetUpdatedAt() && cached.Updated.Before(i.GetUpdatedAt()) {
+			klog.V(1).Infof("cache for %s too old: %s, need %s", key, cached.Updated, i.GetUpdatedAt())
+			h.seen[key] = h.createConversation(i, cs, age)
+			return h.seen[key]
+		}
+
+		if cached.CommentsTotal < len(cs) {
+			klog.V(1).Infof("cache for %s has too few comments: %d, need %d", key, cached.CommentsTotal, len(cs))
+			h.seen[key] = h.createConversation(i, cs, age)
+			return h.seen[key]
+		}
+
+		return h.seen[key]
+	}
+
+	h.seen[key] = h.createConversation(i, cs, age)
+	return h.seen[key]
+}
+
+// createConversation creates a conversation from an issue-like
+func (h *Engine) createConversation(i GitHubItem, cs []*Comment, age time.Time) *Conversation {
+
 	authorIsMember := false
 	if h.isMember(i.GetUser().GetLogin(), i.GetAuthorAssociation()) {
 		authorIsMember = true
@@ -78,6 +103,7 @@ func (h *Engine) conversation(i GitHubItem, cs []*Comment, age time.Time) *Conve
 		Type:                 Issue,
 		Seen:                 age,
 		Created:              i.GetCreatedAt(),
+		Updated:              i.GetUpdatedAt(),
 		CommentsTotal:        i.GetComments(),
 		ClosedAt:             i.GetClosedAt(),
 		SelfInflicted:        authorIsMember,
@@ -212,7 +238,10 @@ func (h *Engine) conversation(i GitHubItem, cs []*Comment, age time.Time) *Conve
 		} else {
 			co.Tags = append(co.Tags, tag.RoleLast(assoc))
 		}
-		co.Updated = last.Updated
+
+		if last.Updated.After(co.Updated) {
+			co.Updated = last.Updated
+		}
 	}
 
 	if co.State == "closed" {

diff --git a/pkg/hubbub/search.go b/pkg/hubbub/search.go
@@ -143,7 +143,6 @@ func (h *Engine) SearchIssues(ctx context.Context, org string, project string, f
 
 		co := h.IssueSummary(i, comments, age)
 		co.Labels = labels
-		h.seen[co.URL] = co
 
 		co.Similar = h.FindSimilar(co)
 		if len(co.Similar) > 0 {
@@ -322,7 +321,6 @@ func (h *Engine) SearchPullRequests(ctx context.Context, org string, project str
 			co.Tags = append(co.Tags, tag.Similar)
 		}
 
-		h.seen[co.URL] = co
 		if !postFetchMatch(co, fs) {
 			klog.V(4).Infof("PR #%d did not pass postFetchMatch with filter: %v", pr.GetNumber(), fs)
 			continue