diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go index 69804b77672..ebc96aac9d8 100644 --- a/modules/migration/retry_downloader.go +++ b/modules/migration/retry_downloader.go @@ -154,6 +154,21 @@ func (d *RetryDownloader) GetComments(ctx context.Context, commentable Commentab return comments, isEnd, err } +func (d *RetryDownloader) GetAllComments(ctx context.Context, page, perPage int) ([]*Comment, bool, error) { + var ( + comments []*Comment + isEnd bool + err error + ) + + err = d.retry(ctx, func(ctx context.Context) error { + comments, isEnd, err = d.Downloader.GetAllComments(ctx, page, perPage) + return err + }) + + return comments, isEnd, err +} + // GetPullRequests returns a repository's pull requests with retry func (d *RetryDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*PullRequest, bool, error) { var ( diff --git a/modules/migration/retry_downloader_test.go b/modules/migration/retry_downloader_test.go index 9b003a7aa56..ade29951a8f 100644 --- a/modules/migration/retry_downloader_test.go +++ b/modules/migration/retry_downloader_test.go @@ -11,12 +11,19 @@ import ( "github.com/stretchr/testify/assert" ) -func TestRetryDownloaderExhaustsAttempts(t *testing.T) { - failure := errors.New("permission denied") - attempts := 0 - assert.ErrorIs(t, NewRetryDownloader(nil, 3, 0).retry(t.Context(), func(_ctx context.Context) error { - attempts++ - return failure - }), failure) - assert.Equal(t, 3, attempts) +type failingCommentsDownloader struct { + NullDownloader + attempts int +} + +func (d *failingCommentsDownloader) GetAllComments(_ctx context.Context, _page, _perPage int) ([]*Comment, bool, error) { + d.attempts++ + return nil, false, errors.New("permission denied") +} + +func TestRetryDownloaderExhaustsAttempts(t *testing.T) { + source := &failingCommentsDownloader{} + _, _, err := NewRetryDownloader(source, 3, 0).GetAllComments(t.Context(), 1, 50) + assert.EqualError(t, err, "permission denied") + assert.Equal(t, 3, source.attempts) } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 6b3ba1eecef..76476cd502e 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -508,15 +508,15 @@ func (g *GiteaLocalUploader) CreateComments(ctx context.Context, comments ...*ba } case issues_model.CommentTypeChangeTitle: if comment.Meta["OldTitle"] != nil { - cm.OldTitle = fmt.Sprint(comment.Meta["OldTitle"]) + cm.OldTitle = util.TruncateRunes(fmt.Sprint(comment.Meta["OldTitle"]), 255) } if comment.Meta["NewTitle"] != nil { - cm.NewTitle = fmt.Sprint(comment.Meta["NewTitle"]) + cm.NewTitle = util.TruncateRunes(fmt.Sprint(comment.Meta["NewTitle"]), 255) } case issues_model.CommentTypeChangeTargetBranch: if comment.Meta["OldRef"] != nil && comment.Meta["NewRef"] != nil { - cm.OldRef = fmt.Sprint(comment.Meta["OldRef"]) - cm.NewRef = fmt.Sprint(comment.Meta["NewRef"]) + cm.OldRef = util.TruncateRunes(fmt.Sprint(comment.Meta["OldRef"]), 255) + cm.NewRef = util.TruncateRunes(fmt.Sprint(comment.Meta["NewRef"]), 255) cm.Content = "" } case issues_model.CommentTypeLabel: @@ -540,7 +540,7 @@ func (g *GiteaLocalUploader) CreateComments(ctx context.Context, comments ...*ba continue } case issues_model.CommentTypeDeleteBranch: - cm.OldRef = comment.Content + cm.OldRef = util.TruncateRunes(comment.Content, 255) cm.Content = "" case issues_model.CommentTypeMergePull: cm.Content = "" @@ -630,7 +630,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *ba } // Anonymous function to download the patch file (allows us to use defer) - err = func() error { + if err := func() error { // if the patchURL is empty there is nothing to download if pr.PatchURL == "" { return nil @@ -656,9 +656,8 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *ba _, err = io.Copy(f, ret) return err - }() - if err != nil { - return "", err + }(); err != nil { // the patch is only a fallback for a missing head ref + log.Warn("PR #%d in %s/%s: unable to download patch: %v", pr.Number, g.repoOwner, g.repoName, err) } head = "unknown repository" diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index 7bb910d0a2f..5175b55dbb8 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -203,6 +203,7 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { &base.Comment{IssueIndex: issue.Index, CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": milestone.Name}}, &base.Comment{IssueIndex: issue.Index, CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": milestone.Name, "Removed": true}}, &base.Comment{IssueIndex: issue.Index, CommentType: "delete_branch", Content: "feature"}, + &base.Comment{IssueIndex: issue.Index, CommentType: "delete_branch", Content: strings.Repeat("a", 256)}, )) unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeLabel, LabelID: label.ID, Content: "1"}) @@ -214,6 +215,7 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeMilestone, MilestoneID: milestone.ID}) unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeMilestone, OldMilestoneID: milestone.ID}) assert.Empty(t, unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeDeleteBranch, OldRef: "feature"}).Content) + assert.Empty(t, unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeDeleteBranch, OldRef: strings.Repeat("a", 255)}).Content) }) } @@ -332,6 +334,7 @@ func TestGiteaUploadForkPullRequestHeadInRepository(t *testing.T) { Head: base.PullRequestBranch{OwnerName: "fork-owner", RepoName: "repo", Ref: "feature", SHA: headCommitID, CloneURL: "https://fork.invalid/fork-owner/repo.git"}, Base: base.PullRequestBranch{OwnerName: repo.OwnerName, RepoName: repo.Name, Ref: repo.DefaultBranch}, } + pr.PatchURL = "file:///nonexistent.patch" pr.EnsuredSafe = true require.True(t, pr.IsForkPullRequest()) diff --git a/services/migrations/github.go b/services/migrations/github.go index e52385bf6ef..ac36bfab4ca 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -167,6 +167,7 @@ func (g *GithubDownloaderV3) waitAndPickClient(ctx context.Context) { err := g.RefreshRate(ctx) if err != nil { log.Error("g.getClient().RateLimit.Get: %s", err) + return // the stale reset time is in the past, so looping would spin } } } @@ -195,14 +196,20 @@ func (g *GithubDownloaderV3) setRate(rate *github.Rate) { g.rates[g.curClientIdx] = rate } +func (g *GithubDownloaderV3) setRateFromResponse(resp *github.Response) { + if resp != nil && resp.Rate.Limit > 0 { + g.setRate(&resp.Rate) + } +} + // GetRepoInfo returns a repository information func (g *GithubDownloaderV3) GetRepoInfo(ctx context.Context) (*base.Repository, error) { g.waitAndPickClient(ctx) gr, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName) + g.setRateFromResponse(resp) if err != nil { return nil, err } - g.setRate(&resp.Rate) // convert github repo to stand Repo return &base.Repository{ @@ -221,10 +228,10 @@ func (g *GithubDownloaderV3) GetRepoInfo(ctx context.Context) (*base.Repository, func (g *GithubDownloaderV3) GetTopics(ctx context.Context) ([]string, error) { g.waitAndPickClient(ctx) r, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName) + g.setRateFromResponse(resp) if err != nil { return nil, err } - g.setRate(&resp.Rate) return r.Topics, nil } @@ -242,10 +249,10 @@ func (g *GithubDownloaderV3) GetMilestones(ctx context.Context) ([]*base.Milesto PerPage: perPage, }, }) + g.setRateFromResponse(resp) if err != nil { return nil, err } - g.setRate(&resp.Rate) for _, m := range ms { state := "open" @@ -288,10 +295,10 @@ func (g *GithubDownloaderV3) GetLabels(ctx context.Context) ([]*base.Label, erro Page: i, PerPage: perPage, }) + g.setRateFromResponse(resp) if err != nil { return nil, err } - g.setRate(&resp.Rate) for _, label := range ls { labels = append(labels, convertGithubLabel(label)) @@ -387,10 +394,10 @@ func (g *GithubDownloaderV3) GetReleases(ctx context.Context) ([]*base.Release, Page: i, PerPage: perPage, }) + g.setRateFromResponse(resp) if err != nil { return nil, err } - g.setRate(&resp.Rate) for _, release := range ls { releases = append(releases, g.convertGithubRelease(ctx, release)) @@ -429,49 +436,20 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ( allIssues := make([]*base.Issue, 0, perPage) g.waitAndPickClient(ctx) issues, resp, err := g.getClient().Issues.ListByRepo(ctx, g.repoOwner, g.repoName, opt) + g.setRateFromResponse(resp) if err != nil { return nil, false, fmt.Errorf("GetIssues: %w", err) } log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues)) - g.setRate(&resp.Rate) for _, issue := range issues { if issue.IsPullRequest() { continue } - - labels := make([]*base.Label, 0, len(issue.Labels)) - for _, l := range issue.Labels { - labels = append(labels, convertGithubLabel(l)) - } - - reactions, err := g.getIssueReactions(ctx, issue.GetNumber(), perPage) + converted, err := g.convertGithubIssue(ctx, issue, perPage) if err != nil { return nil, false, err } - - var assignees []string - for i := range issue.Assignees { - assignees = append(assignees, issue.Assignees[i].GetLogin()) - } - - allIssues = append(allIssues, &base.Issue{ - Title: *issue.Title, - Number: int64(*issue.Number), - PosterID: issue.GetUser().GetID(), - PosterName: issue.GetUser().GetLogin(), - PosterEmail: issue.GetUser().GetEmail(), - Content: issue.GetBody(), - Milestone: issue.GetMilestone().GetTitle(), - State: issue.GetState(), - Created: issue.GetCreatedAt().Time, - Updated: issue.GetUpdatedAt().Time, - Labels: labels, - Reactions: reactions, - Closed: issue.ClosedAt.GetTime(), - IsLocked: issue.GetLocked(), - Assignees: assignees, - ForeignIndex: int64(*issue.Number), - }) + allIssues = append(allIssues, converted) } g.issuePage, g.issueCursor, g.nextIssueCursor = page, opt.After, resp.After @@ -485,6 +463,82 @@ func (g *GithubDownloaderV3) isLastPage(resp *github.Response, n, perPage int) b return resp.NextPage == 0 && resp.After == "" } +func (g *GithubDownloaderV3) convertGithubIssue(ctx context.Context, issue *github.Issue, perPage int) (*base.Issue, error) { + labels := make([]*base.Label, 0, len(issue.Labels)) + for _, l := range issue.Labels { + labels = append(labels, convertGithubLabel(l)) + } + + // get reactions + var reactions []*base.Reaction + if !g.SkipReactions && (issue.Reactions == nil || issue.Reactions.TotalCount == nil || issue.Reactions.GetTotalCount() > 0) { + for i := 1; ; i++ { + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: perPage, + }, + }) + g.setRateFromResponse(resp) + if err != nil { + return nil, err + } + if len(res) == 0 { + break + } + for _, reaction := range res { + reactions = append(reactions, &base.Reaction{ + UserID: reaction.User.GetID(), + UserName: reaction.User.GetLogin(), + Content: reaction.GetContent(), + }) + } + } + } + + var assignees []string + for i := range issue.Assignees { + assignees = append(assignees, issue.Assignees[i].GetLogin()) + } + + return &base.Issue{ + Title: issue.GetTitle(), + Number: int64(issue.GetNumber()), + PosterID: issue.GetUser().GetID(), + PosterName: issue.GetUser().GetLogin(), + PosterEmail: issue.GetUser().GetEmail(), + Content: issue.GetBody(), + Milestone: issue.GetMilestone().GetTitle(), + State: issue.GetState(), + Created: issue.GetCreatedAt().Time, + Updated: issue.GetUpdatedAt().Time, + Labels: labels, + Reactions: reactions, + Closed: issue.ClosedAt.GetTime(), + IsLocked: issue.GetLocked(), + Assignees: assignees, + ForeignIndex: int64(issue.GetNumber()), + }, nil +} + +// GetIssue recovers an issue omitted from the repository listing. +func (g *GithubDownloaderV3) GetIssue(ctx context.Context, number int64) (*base.Issue, error) { + g.waitAndPickClient(ctx) + issue, resp, err := g.getClient().Issues.Get(ctx, g.repoOwner, g.repoName, int(number)) + g.setRateFromResponse(resp) + if err != nil { + return nil, err + } + if issue.IsPullRequest() { + return nil, fmt.Errorf("issue %d is a pull request", number) + } + if int64(issue.GetNumber()) != number { + return nil, fmt.Errorf("GitHub returned issue %d while recovering issue %d", issue.GetNumber(), number) + } + return g.convertGithubIssue(ctx, issue, g.maxPerPage) +} + // SupportGetRepoComments return true if it supports get repo comments func (g *GithubDownloaderV3) SupportGetRepoComments() bool { return true @@ -512,14 +566,14 @@ func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.C for { g.waitAndPickClient(ctx) comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt) + g.setRateFromResponse(resp) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } - g.setRate(&resp.Rate) for _, comment := range comments { // get reactions var reactions []*base.Reaction - if !g.SkipReactions { + if !g.SkipReactions && mayHaveReactions(comment.Reactions) { for i := 1; ; i++ { g.waitAndPickClient(ctx) res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{ @@ -528,10 +582,10 @@ func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.C PerPage: g.maxPerPage, }, }) + g.setRateFromResponse(resp) if err != nil { return nil, err } - g.setRate(&resp.Rate) if len(res) == 0 { break } @@ -586,17 +640,17 @@ func (g *GithubDownloaderV3) GetAllComments(ctx context.Context, page, perPage i g.waitAndPickClient(ctx) comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, 0, opt) + g.setRateFromResponse(resp) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } isEnd := resp.NextPage == 0 log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage) - g.setRate(&resp.Rate) for _, comment := range comments { // get reactions var reactions []*base.Reaction - if !g.SkipReactions { + if !g.SkipReactions && mayHaveReactions(comment.Reactions) { for i := 1; ; i++ { g.waitAndPickClient(ctx) res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{ @@ -605,10 +659,10 @@ func (g *GithubDownloaderV3) GetAllComments(ctx context.Context, page, perPage i PerPage: g.maxPerPage, }, }) + g.setRateFromResponse(resp) if err != nil { return nil, false, err } - g.setRate(&resp.Rate) if len(res) == 0 { break } @@ -656,11 +710,11 @@ func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage allPRs := make([]*base.PullRequest, 0, perPage) g.waitAndPickClient(ctx) prs, resp, err := g.getClient().PullRequests.List(ctx, g.repoOwner, g.repoName, opt) + g.setRateFromResponse(resp) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs)) - g.setRate(&resp.Rate) for _, pr := range prs { basePR, err := g.convertGithubPullRequest(ctx, pr, perPage) if err != nil { @@ -681,7 +735,7 @@ func (g *GithubDownloaderV3) convertGithubPullRequest(ctx context.Context, pr *g labels = append(labels, convertGithubLabel(l)) } - reactions, err := g.getIssueReactions(ctx, pr.GetNumber(), perPage) + reactions, err := g.getIssueReactions(ctx, pr.GetNumber(), perPage, nil) if err != nil { return nil, err } @@ -725,8 +779,12 @@ func (g *GithubDownloaderV3) convertGithubPullRequest(ctx context.Context, pr *g }, nil } -func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perPage int) ([]*base.Reaction, error) { - if g.SkipReactions { +func mayHaveReactions(reactions *github.Reactions) bool { + return reactions == nil || reactions.TotalCount == nil || *reactions.TotalCount > 0 +} + +func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perPage int, rollup *github.Reactions) ([]*base.Reaction, error) { + if g.SkipReactions || !mayHaveReactions(rollup) { return nil, nil } var reactions []*base.Reaction @@ -738,10 +796,10 @@ func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perP PerPage: perPage, }, }) + g.setRateFromResponse(resp) if err != nil { return nil, err } - g.setRate(&resp.Rate) if len(res) == 0 { return reactions, nil } @@ -774,7 +832,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(ctx context.Context, cs for _, c := range cs { // get reactions var reactions []*base.Reaction - if !g.SkipReactions { + if !g.SkipReactions && mayHaveReactions(c.Reactions) { for i := 1; ; i++ { g.waitAndPickClient(ctx) res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListReactionOptions{ @@ -783,10 +841,10 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(ctx context.Context, cs PerPage: g.maxPerPage, }, }) + g.setRateFromResponse(resp) if err != nil { return nil, err } - g.setRate(&resp.Rate) if len(res) == 0 { break } @@ -830,10 +888,10 @@ func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Rev for { g.waitAndPickClient(ctx) reviews, resp, err := g.getClient().PullRequests.ListReviews(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) + g.setRateFromResponse(resp) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } - g.setRate(&resp.Rate) for _, review := range reviews { r := convertGithubReview(review) r.IssueIndex = reviewable.GetLocalIndex() @@ -844,10 +902,10 @@ func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Rev for { g.waitAndPickClient(ctx) reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) + g.setRateFromResponse(resp) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } - g.setRate(&resp.Rate) cs, err := g.convertGithubReviewComments(ctx, reviewComments) if err != nil { @@ -869,10 +927,10 @@ func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Rev // Get requested reviews g.waitAndPickClient(ctx) reviewers, resp, err := g.getClient().PullRequests.ListReviewers(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex())) + g.setRateFromResponse(resp) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } - g.setRate(&resp.Rate) for _, user := range reviewers.Users { r := &base.Review{ ReviewerID: user.GetID(), diff --git a/services/migrations/github_issue_recovery.go b/services/migrations/github_issue_recovery.go new file mode 100644 index 00000000000..9882a38b759 --- /dev/null +++ b/services/migrations/github_issue_recovery.go @@ -0,0 +1,70 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "context" + "fmt" + + base "gitea.dev/modules/migration" +) + +type githubIssueRecoveryUploader struct { + base.Uploader + downloader *GithubDownloaderV3 + issues map[int64]struct{} +} + +func wrapGithubIssueRecoveryUploader(uploader base.Uploader, downloader base.Downloader) base.Uploader { + if retry, ok := downloader.(*base.RetryDownloader); ok { + downloader = retry.Downloader + } + github, ok := downloader.(*GithubDownloaderV3) + if !ok { + return uploader + } + return &githubIssueRecoveryUploader{ + Uploader: uploader, + downloader: github, + issues: make(map[int64]struct{}), + } +} + +func (g *githubIssueRecoveryUploader) CreateIssues(ctx context.Context, issues ...*base.Issue) error { + if err := g.Uploader.CreateIssues(ctx, issues...); err != nil { + return err + } + for _, issue := range issues { + g.issues[issue.Number] = struct{}{} + } + return nil +} + +func (g *githubIssueRecoveryUploader) CreatePullRequests(ctx context.Context, prs ...*base.PullRequest) error { + if err := g.Uploader.CreatePullRequests(ctx, prs...); err != nil { + return err + } + for _, pr := range prs { + g.issues[pr.Number] = struct{}{} + } + return nil +} + +func (g *githubIssueRecoveryUploader) CreateComments(ctx context.Context, comments ...*base.Comment) error { + for _, comment := range comments { + if _, ok := g.issues[comment.IssueIndex]; ok { + continue + } + + issue, err := g.downloader.GetIssue(ctx, comment.IssueIndex) + if err != nil { + return fmt.Errorf("recover missing IssueIndex %d: %w", comment.IssueIndex, err) + } + if err := g.Uploader.CreateIssues(ctx, issue); err != nil { + return err + } + g.issues[issue.Number] = struct{}{} + } + return g.Uploader.CreateComments(ctx, comments...) +} diff --git a/services/migrations/github_test.go b/services/migrations/github_test.go index c8a0a32b948..1ba8b8e8bc1 100644 --- a/services/migrations/github_test.go +++ b/services/migrations/github_test.go @@ -120,6 +120,57 @@ func TestGithubIssuePagination(t *testing.T) { } } +func TestGithubGetAllCommentsSkipsEmptyReactionsAndRecordsRateLimit(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/v3/repos/o/r/issues/comments", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-RateLimit-Limit", "5000") + w.Header().Set("X-RateLimit-Reset", "4102444800") + if r.URL.Query().Get("page") == "1" { + w.Header().Set("X-RateLimit-Remaining", "100") + _, _ = w.Write([]byte(`[{"issue_url":"https://api.github.com/repos/o/r/issues/7","reactions":{"total_count":0}}]`)) + return + } + + w.Header().Set("X-RateLimit-Remaining", "0") + w.WriteHeader(http.StatusForbidden) + })) + defer server.Close() + + downloader, err := NewGithubDownloaderV3(t.Context(), server.URL, "", "", "", "o", "r") + require.NoError(t, err) + comments, _, err := downloader.GetAllComments(t.Context(), 1, 100) + require.NoError(t, err) + require.Len(t, comments, 1) + + _, _, err = downloader.GetAllComments(t.Context(), 2, 100) + require.Error(t, err) + assert.Zero(t, downloader.rates[0].Remaining) +} + +func TestGithubGetIssueRecoversIssueOmittedFromList(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v3/repos/o/r/issues/123": + _, _ = w.Write([]byte(`{"number":123,"title":"recovered issue","user":{"id":1,"login":"user"},"created_at":"2024-08-12T08:38:46Z","updated_at":"2025-01-21T10:05:11Z","state":"closed"}`)) + case "/api/v3/repos/o/r/issues/123/reactions": + _, _ = w.Write([]byte(`[]`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + downloader, err := NewGithubDownloaderV3(t.Context(), server.URL, "", "", "", "o", "r") + require.NoError(t, err) + + issue, err := downloader.GetIssue(t.Context(), 123) + require.NoError(t, err) + assert.EqualValues(t, 123, issue.Number) + assert.Equal(t, "recovered issue", issue.Title) +} + func TestGitHubDownloadRepo(t *testing.T) { token := os.Getenv("GITHUB_READ_TOKEN") liveMode := token != "" diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index ed7c13209ef..b2525c26e37 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -109,7 +109,7 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader.gitServiceType = opts.GitServiceType - if err := migrateRepository(ctx, doer, downloader, uploader, opts, messenger); err != nil { + if err := migrateRepository(ctx, doer, downloader, wrapGithubIssueRecoveryUploader(uploader, downloader), opts, messenger); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) }