diff --git a/modules/storage/azureblob.go b/modules/storage/azureblob.go index acf60972582..dcc274e5dc8 100644 --- a/modules/storage/azureblob.go +++ b/modules/storage/azureblob.go @@ -304,12 +304,10 @@ func (a *AzureBlobStorage) ServeDirectURL(storePath, name, method string, reqPar // IterateObjects iterates across the objects in the azureblobstorage func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error { - dirName = a.buildAzureBlobPath(dirName) - if dirName != "" { - dirName += "/" - } + basePrefix := buildObjectStorePathPrefix(a.cfg.BasePath, "") + dirPrefix := buildObjectStorePathPrefix(a.cfg.BasePath, dirName) pager := a.client.NewListBlobsFlatPager(a.cfg.Container, &container.ListBlobsFlatOptions{ - Prefix: &dirName, + Prefix: &dirPrefix, }) for pager.More() { resp, err := pager.NextPage(a.ctx) @@ -317,7 +315,8 @@ func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, o return convertAzureBlobErr(err) } for _, object := range resp.Segment.BlobItems { - blobClient := a.getBlobClient(*object.Name) + objPath := strings.TrimPrefix(*object.Name, basePrefix) + blobClient := a.getBlobClient(objPath) object := &azureBlobObject{ Context: a.ctx, blobClient: blobClient, @@ -327,7 +326,7 @@ func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, o } if err := func(object *azureBlobObject, fn func(path string, obj Object) error) error { defer object.Close() - return fn(strings.TrimPrefix(object.Name, a.cfg.BasePath), object) + return fn(objPath, object) }(object, fn); err != nil { return convertAzureBlobErr(err) } @@ -336,7 +335,6 @@ func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, o return nil } -// Delete delete a file func (a *AzureBlobStorage) getBlobClient(path string) *blob.Client { return a.client.ServiceClient().NewContainerClient(a.cfg.Container).NewBlobClient(a.buildAzureBlobPath(path)) } diff --git a/modules/storage/azureblob_test.go b/modules/storage/azureblob_test.go index 6f184ea0058..501f68c3afd 100644 --- a/modules/storage/azureblob_test.go +++ b/modules/storage/azureblob_test.go @@ -27,24 +27,16 @@ func TestAzureBlobStorage(t *testing.T) { Container: "test", }, } - table := []struct { - name string - test func(t *testing.T, typStr Type, cfg *setting.Storage) - }{ - { - name: "iterator", - test: testStorageIterator, - }, - { - name: "testBlobStorageURLContentTypeAndDisposition", - test: testBlobStorageURLContentTypeAndDisposition, - }, - } - for _, entry := range table { - t.Run(entry.name, func(t *testing.T) { - entry.test(t, storageType, config) - }) - } + t.Run("Iterator", func(t *testing.T) { + testStorageIterator(t, storageType, config) + }) + t.Run("BlobStorageURLContentTypeAndDisposition", func(t *testing.T) { + testBlobStorageURLContentTypeAndDisposition(t, storageType, config) + }) + t.Run("IteratorWithBasePath", func(t *testing.T) { + config.AzureBlobConfig.BasePath = "test-base-path" + testStorageIterator(t, storageType, config) + }) } func TestAzureBlobStoragePath(t *testing.T) { diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 43c79f5b35c..ba3058d333a 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -11,11 +11,13 @@ import ( "net/url" "os" "path" + "strings" "gitea.dev/modules/httplib" "gitea.dev/modules/log" "gitea.dev/modules/public" "gitea.dev/modules/setting" + "gitea.dev/modules/util" ) // ErrURLNotSupported represents url is not supported @@ -139,6 +141,23 @@ func SaveFrom(objStorage ObjectStorage, path string, callback func(w io.Writer) return err } +func buildObjectStorePath(base, p string) string { + p = strings.TrimPrefix(util.PathJoinRelX(base, p), "/") // object store doesn't use slash for root path + if p == "." { + p = "" // object store doesn't use dot as relative path + } + return p +} + +func buildObjectStorePathPrefix(base, p string) string { + // ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo" + p = buildObjectStorePath(base, p) + "/" + if p == "/" { + p = "" // object store doesn't use slash for root path + } + return p +} + var ( // Attachments represents attachments storage Attachments ObjectStorage = uninitializedStorage diff --git a/modules/storage/storage_test.go b/modules/storage/storage_test.go index 003b1a9e8a8..b40e5d8f8e4 100644 --- a/modules/storage/storage_test.go +++ b/modules/storage/storage_test.go @@ -4,6 +4,7 @@ package storage import ( + "io" "net/http" "strings" "testing" @@ -31,6 +32,11 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) { _, err = l.Save(f[0], strings.NewReader(f[1]), -1) assert.NoError(t, err) } + defer func() { + for _, f := range testFiles { + _ = l.Delete(f[0]) + } + }() expectedList := map[string][]string{ "a": {"a/1.txt"}, @@ -43,7 +49,9 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) { for dir, expected := range expectedList { count := 0 err = l.IterateObjects(dir, func(path string, f Object) error { - defer f.Close() + content, err := io.ReadAll(f) + assert.NoError(t, err) + assert.NotEmpty(t, content) assert.Contains(t, expected, path) count++ return nil