fix(storage): fix Azure Blob dump failing with file does not exist (#38814)

## Issue Gitea fails to dump LFS (and other object-storage) files when Azure Blob Storage is configured as the storage backend. The dump reports: Failed to dump LFS objects: /file/path: copying contents: file does not exist This happens with any non-empty base path (the default for LFS storage), which is why the user could only work around it by using `--skip-*` flags. The root cause is in `AzureBlobStorage.IterateObjects()`: Azure's list API already returns each blob's name including the configured base path, but the code was building the read client by running that name through the base-path-prepending helper a second time. This doubled the base path (e.g. `gitea-lfs/gitea-lfs/aa/bb/hash`), pointing at a blob that doesn't exist. `Stat()` still succeeded because it doesn't touch the network, so the failure only surfaced when the dumper actually tried to read the object's contents. ## Solution Add `getBlobClientByFullName()`, which builds a blob client from a name that is already fully qualified, without re-applying `buildAzureBlobPath()`. `IterateObjects()` now uses it for names obtained from Azure's list API. `getBlobClient()` (used by `Open`, `Stat`, `Delete`, `ServeDirectURL`, which take relative paths) is unchanged in behavior. Also add `TestAzureBlobStorageDumpArchive`, a regression test that drives the real dump path (`IterateObjects` → `Stat` → `dump.Dumper.AddFileByReader` → `mholt/archives` zip writer) against a **non-empty** `BasePath`, and verifies the produced archive contains the object with the correct content. The existing Azure tests use an empty `BasePath` and never read object content via `IterateObjects`, which is why they didn't catch this. Fixes #35476 --------- Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
2026-08-09 03:30:25 +00:00 · 2026-08-07 22:05:30 +05:30
parent 4c382cea59
commit 7733f1953f
7 changed files with 198 additions and 226 deletions
@@ -4,20 +4,50 @@
 package storage

 import (
+	"io"
 	"net/http"
 	"strings"
 	"testing"

-	"gitea.dev/modules/setting"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

-func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) {
-	l, err := NewStorage(typStr, cfg)
-	assert.NoError(t, err)
+func TestObjectStoragePath(t *testing.T) {
+	base := ""
+	assert.Empty(t, buildObjectStorePath(base, "/"))
+	assert.Empty(t, buildObjectStorePath(base, "."))
+	assert.Equal(t, "a", buildObjectStorePath(base, "/a"))
+	assert.Equal(t, "a/b", buildObjectStorePath(base, "/a/b/"))
+	assert.Empty(t, buildObjectStorePathPrefix(base, ""))
+	assert.Equal(t, "a/", buildObjectStorePathPrefix(base, "/a/"))

+	base = "/"
+	assert.Empty(t, buildObjectStorePath(base, "/"))
+	assert.Empty(t, buildObjectStorePath(base, "."))
+	assert.Equal(t, "a", buildObjectStorePath(base, "/a"))
+	assert.Equal(t, "a/b", buildObjectStorePath(base, "/a/b/"))
+	assert.Empty(t, buildObjectStorePathPrefix(base, ""))
+	assert.Equal(t, "a/", buildObjectStorePathPrefix(base, "/a/"))
+
+	base = "/base"
+	assert.Equal(t, "base", buildObjectStorePath(base, "/"))
+	assert.Equal(t, "base", buildObjectStorePath(base, "."))
+	assert.Equal(t, "base/a", buildObjectStorePath(base, "/a"))
+	assert.Equal(t, "base/a/b", buildObjectStorePath(base, "/a/b/"))
+	assert.Equal(t, "base/", buildObjectStorePathPrefix(base, ""))
+	assert.Equal(t, "base/a/", buildObjectStorePathPrefix(base, "/a/"))
+
+	base = "/base/"
+	assert.Equal(t, "base", buildObjectStorePath(base, "/"))
+	assert.Equal(t, "base", buildObjectStorePath(base, "."))
+	assert.Equal(t, "base/a", buildObjectStorePath(base, "/a"))
+	assert.Equal(t, "base/a/b", buildObjectStorePath(base, "/a/b/"))
+	assert.Equal(t, "base/", buildObjectStorePathPrefix(base, ""))
+	assert.Equal(t, "base/a/", buildObjectStorePathPrefix(base, "/a/"))
+}
+
+func testStorageIterator(t *testing.T, objStore ObjectStorage) {
 	testFiles := [][]string{
 		{"a/1.txt", "a1"},
 		{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
@@ -28,12 +58,19 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) {
 		{"b/x 4.txt", "bx4"},
 	}
 	for _, f := range testFiles {
-		_, err = l.Save(f[0], strings.NewReader(f[1]), -1)
+		_, err := objStore.Save(f[0], strings.NewReader(f[1]), -1)
 		assert.NoError(t, err)
 	}
+	defer func() {
+		for _, f := range testFiles {
+			_ = objStore.Delete(f[0])
+		}
+	}()

 	expectedList := map[string][]string{
 		"a":           {"a/1.txt"},
+		"a/":          {"a/1.txt"},
+		"/a/":         {"a/1.txt"},
 		"b":           {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
 		"":            {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
 		"/":           {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
@@ -42,8 +79,10 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) {
 	}
 	for dir, expected := range expectedList {
 		count := 0
-		err = l.IterateObjects(dir, func(path string, f Object) error {
-			defer f.Close()
+		err := objStore.IterateObjects(dir, func(path string, f Object) error {
+			content, err := io.ReadAll(f)
+			assert.NoError(t, err)
+			assert.NotEmpty(t, content)
 			assert.Contains(t, expected, path)
 			count++
 			return nil
@@ -53,52 +92,57 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) {
 	}
 }

-type expectedServeDirectHeaders struct {
-	ContentType        string
-	ContentDisposition string
-}
-
-func testSingleBlobStorageURLContentTypeAndDisposition(t *testing.T, s ObjectStorage, path, name string, expected expectedServeDirectHeaders, reqParams *ServeDirectOptions) {
-	u, err := s.ServeDirectURL(path, name, http.MethodGet, reqParams)
-	require.NoError(t, err)
-	resp, err := http.Get(u.String())
-	require.NoError(t, err)
-	defer resp.Body.Close()
-	if expected.ContentType != "" {
-		assert.Equal(t, expected.ContentType, resp.Header.Get("Content-Type"))
+func testStorageURLContentTypeAndDisposition(t *testing.T, objStore ObjectStorage) {
+	type expectedServeDirectHeaders struct {
+		ContentType        string
+		ContentDisposition string
 	}
-	if expected.ContentDisposition != "" {
-		assert.Equal(t, expected.ContentDisposition, resp.Header.Get("Content-Disposition"))
+	test := func(t *testing.T, s ObjectStorage, path, name string, expected expectedServeDirectHeaders, reqParams *ServeDirectOptions) {
+		u, err := s.ServeDirectURL(path, name, http.MethodGet, reqParams)
+		require.NoError(t, err)
+		resp, err := http.Get(u.String())
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		if expected.ContentType != "" {
+			assert.Equal(t, expected.ContentType, resp.Header.Get("Content-Type"))
+		}
+		if expected.ContentDisposition != "" {
+			assert.Equal(t, expected.ContentDisposition, resp.Header.Get("Content-Disposition"))
+		}
 	}
-}
-
-func testBlobStorageURLContentTypeAndDisposition(t *testing.T, typStr Type, cfg *setting.Storage) {
-	s, err := NewStorage(typStr, cfg)
-	assert.NoError(t, err)

 	testFilename := "test.txt"
-	_, err = s.Save(testFilename, strings.NewReader("dummy-content"), -1)
+	_, err := objStore.Save(testFilename, strings.NewReader("dummy-content"), -1)
 	assert.NoError(t, err)

-	testSingleBlobStorageURLContentTypeAndDisposition(t, s, testFilename, "test.txt", expectedServeDirectHeaders{
+	test(t, objStore, testFilename, "test.txt", expectedServeDirectHeaders{
 		ContentType:        "text/plain; charset=utf-8",
 		ContentDisposition: `inline; filename=test.txt`,
 	}, nil)

-	testSingleBlobStorageURLContentTypeAndDisposition(t, s, testFilename, "test.pdf", expectedServeDirectHeaders{
+	test(t, objStore, testFilename, "test.pdf", expectedServeDirectHeaders{
 		ContentType:        "application/pdf",
 		ContentDisposition: `inline; filename=test.pdf`,
 	}, nil)

-	testSingleBlobStorageURLContentTypeAndDisposition(t, s, testFilename, "test.wasm", expectedServeDirectHeaders{
+	test(t, objStore, testFilename, "test.wasm", expectedServeDirectHeaders{
 		ContentDisposition: `inline; filename=test.wasm`,
 	}, nil)

-	testSingleBlobStorageURLContentTypeAndDisposition(t, s, testFilename, "test.wasm", expectedServeDirectHeaders{
+	test(t, objStore, testFilename, "test.wasm", expectedServeDirectHeaders{
 		ContentType:        "application/wasm",
 		ContentDisposition: `inline; filename=test.wasm`,
 	}, &ServeDirectOptions{
 		ContentType: "application/wasm",
 	})
-	assert.NoError(t, s.Delete(testFilename))
+	assert.NoError(t, objStore.Delete(testFilename))
+}
+
+func testStorageGeneral(t *testing.T, objStore ObjectStorage) {
+	t.Run("StorageIterator", func(t *testing.T) { testStorageIterator(t, objStore) })
+
+	if _, ok := objStore.(*LocalStorage); ok {
+		t.Skipf("Skipping tests for local storage")
+	}
+	t.Run("StorageURLContentTypeAndDisposition", func(t *testing.T) { testStorageURLContentTypeAndDisposition(t, objStore) })
 }