Tap: prioritize pending repos first (#1329)

When picking up "resync" jobs, process them in the following order:
pending, desynchronized, error.

In other words:
- focus on getting out of "backfill" mode
- then focus on fixing repos that have gotten desynchronized but that we
haven't failed to resync
- only after all those are done, go back and try to fix repos that had
an error during resync.

By default, tap was selecting desyncrhonized repos first which
dramatically reduced tap's throughput and made it look like tap had
stopped crawling the full network.

I believe this addresses the issue from:
#1315

Author: dholms

cmd/tap/resyncer.go

@@ -98,19 +98,34 @@ func (r *Resyncer) claimResyncJob(ctx context.Context) (string, bool, error) {
 	r.claimJobMu.Lock()
 	defer r.claimJobMu.Unlock()
 
+	// prioritize pending repos first before moving on to desyncrhonized
+	// only reprocess repos that errored after resyncing all other repos
+	for _, state := range []models.RepoState{models.RepoStatePending, models.RepoStateDesynchronized, models.RepoStateError} {
+		did, found, err := r.claimResyncJobOfState(ctx, state)
+		if err != nil {
+			return "", false, err
+		}
+		if found {
+			return did, true, nil
+		}
+	}
+	return "", false, nil
+}
+
+func (r *Resyncer) claimResyncJobOfState(ctx context.Context, state models.RepoState) (string, bool, error) {
 	var did string
 	now := time.Now().Unix()
 	result := r.db.WithContext(ctx).Raw(`
 		UPDATE repos
 		SET state = ?
 		WHERE did = (
 			SELECT did FROM repos
-			WHERE state IN (?, ?, ?)
+			WHERE state = ?
 			AND (retry_after = 0 OR retry_after < ?)
 			LIMIT 1
 		)
 		RETURNING did
-		`, models.RepoStateResyncing, models.RepoStatePending, models.RepoStateDesynchronized, models.RepoStateError, now).Scan(&did)
+		`, models.RepoStateResyncing, state, now).Scan(&did)
 	if result.Error != nil {
 		return "", false, result.Error
 	}