// Copyright 2024 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package repository import ( "context" "fmt" "io" "path" "strings" "gitea.dev/models/db" repo_model "gitea.dev/models/repo" "gitea.dev/modules/container" "gitea.dev/modules/git" "gitea.dev/modules/graceful" "gitea.dev/modules/log" "gitea.dev/modules/options" "gitea.dev/modules/queue" "gitea.dev/modules/util" licenseclassifier "github.com/google/licenseclassifier/v2" ) const ( LicenseLegacyFile = "LICENSE" // LicenseReuseDir is for REUSE license spec - see https://reuse.software/spec-3.3/ // TODO: Surface this version in repo creation LicenseReuseDir = "LICENSES" ) var ( classifier *licenseclassifier.Classifier // licenseUpdaterQueue represents a queue to handle update repo licenses licenseUpdaterQueue *queue.WorkerPoolQueue[*LicenseUpdaterOptions] ) func AddRepoToLicenseUpdaterQueue(opts *LicenseUpdaterOptions) error { if opts == nil { return nil } return licenseUpdaterQueue.Push(opts) } func InitLicenseClassifier() error { // threshold should be 0.84~0.86 or the test will be failed classifier = licenseclassifier.NewClassifier(.85) licenseFiles, err := options.AssetFS().ListFiles("license", true) if err != nil { return err } for _, licenseFile := range licenseFiles { licenseName := licenseFile data, err := options.License(licenseFile) if err != nil { return err } classifier.AddContent("License", licenseName, licenseName, data) } return nil } type LicenseUpdaterOptions struct { RepoID int64 } func repoLicenseUpdater(items ...*LicenseUpdaterOptions) []*LicenseUpdaterOptions { ctx := graceful.GetManager().ShutdownContext() for _, opts := range items { repo, err := repo_model.GetRepositoryByID(ctx, opts.RepoID) if err != nil { log.Error("repoLicenseUpdater [%d] failed: GetRepositoryByID: %v", opts.RepoID, err) continue } if repo.IsEmpty { continue } func() { gitRepo, err := git.OpenRepository(ctx, repo) if err != nil { log.Error("repoLicenseUpdater [%d] failed: OpenRepository: %v", opts.RepoID, err) return } defer gitRepo.Close() commit, err := gitRepo.GetBranchCommit(ctx, repo.DefaultBranch) if err != nil { log.Error("repoLicenseUpdater [%d] failed: GetBranchCommit: %v", opts.RepoID, err) return } if err = UpdateRepoLicenses(ctx, repo, gitRepo, commit); err != nil { log.Error("repoLicenseUpdater [%d] failed: updateRepoLicenses: %v", opts.RepoID, err) } }() } return nil } func SyncRepoLicenses(ctx context.Context) error { log.Trace("Doing: SyncRepoLicenses") if err := db.Iterate( ctx, nil, func(ctx context.Context, repo *repo_model.Repository) error { select { case <-ctx.Done(): return db.ErrCancelledf("before sync repo licenses for %s", repo.FullName()) default: } return AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{RepoID: repo.ID}) }, ); err != nil { log.Trace("Error: SyncRepoLicenses: %v", err) return err } log.Trace("Finished: SyncReposLicenses") return nil } // resolveReuseLicenses gathers all licenses in a subtree (assumed to be LicenseReuseDir as per REUSE specification) func resolveReuseLicenses(ctx context.Context, gitrepo *git.Repository, parentPath string, tree *git.Tree) ([]repo_model.DetectedLicense, error) { entries, err := tree.ListEntries(ctx, gitrepo) if err != nil { return nil, fmt.Errorf("ListEntries: %w", err) } licenses := make([]repo_model.DetectedLicense, 0) for _, entry := range entries { if entry.IsRegular() { spdxID := util.PathBaseStem(entry.Name()) licenses = append(licenses, repo_model.DetectedLicense{SPDXID: spdxID, LicensePath: path.Join(parentPath, entry.Name())}) } } return licenses, nil } // isLicenseFile checks the prefix of the file and determines if it could plausibly be a license one // it's checking well-known ones: license, licence and copying // allowed extensions are: md, lesser and txt func isLicenseFile(name string) bool { lower := strings.ToLower(name) stem := util.PathBaseStem(lower) ext := path.Ext(lower) // exact match (e.g. "LICENSE") or at most one allowed extension (e.g. "LICENSE.md") return (stem == "license" || stem == "licence" || stem == "copying") && (ext == "" || ext == ".md" || ext == ".lesser" || ext == ".txt") } func resolveLicenses(ctx context.Context, gitRepo *git.Repository, commit *git.Commit) ([]repo_model.DetectedLicense, error) { tree, err := commit.SubTree(ctx, gitRepo, LicenseReuseDir) if err != nil && !git.IsErrNotExist(err) { return nil, fmt.Errorf("SubTree: %w", err) } // handle REUSE license spec first if !git.IsErrNotExist(err) { return resolveReuseLicenses(ctx, gitRepo, LicenseReuseDir, tree) } tree, err = commit.SubTree(ctx, gitRepo, "") if err != nil && !git.IsErrNotExist(err) { return nil, fmt.Errorf("SubTree: %w", err) } entries, err := tree.ListEntries(ctx, gitRepo) if err != nil { return nil, fmt.Errorf("ListEntries: %w", err) } licenses := make([]repo_model.DetectedLicense, 0) for _, entry := range entries { if !entry.IsRegular() { continue } if !isLicenseFile(entry.Name()) { continue } r, err := entry.Blob(gitRepo).DataAsync(ctx) if err != nil { continue } found, err := detectLicense(r) _ = r.Close() if err != nil { continue } for _, license := range found { licenses = append(licenses, repo_model.DetectedLicense{SPDXID: license, LicensePath: entry.Name()}) } } return licenses, nil } // UpdateRepoLicenses will update repository licenses col if license file exists func UpdateRepoLicenses(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, commit *git.Commit) error { licenses, err := resolveLicenses(ctx, gitRepo, commit) if err != nil { return err } if len(licenses) == 0 { return repo_model.CleanRepoLicenses(ctx, repo) } return repo_model.UpdateRepoLicenses(ctx, repo, commit.ID.String(), licenses) } // detectLicense returns the licenses detected by the given content buff func detectLicense(r io.Reader) ([]string, error) { matches, err := classifier.MatchFrom(r) if err != nil { return nil, err } if len(matches.Matches) > 0 { results := make(container.Set[string], len(matches.Matches)) for _, r := range matches.Matches { if r.MatchType == "License" && !results.Contains(r.Variant) { results.Add(r.Variant) } } return results.Values(), nil } return nil, nil }