mirror of
https://github.com/go-gitea/gitea.git
synced 2026-08-08 12:27:44 +00:00
7733f1953f
## Issue Gitea fails to dump LFS (and other object-storage) files when Azure Blob Storage is configured as the storage backend. The dump reports: Failed to dump LFS objects: /file/path: copying contents: file does not exist This happens with any non-empty base path (the default for LFS storage), which is why the user could only work around it by using `--skip-*` flags. The root cause is in `AzureBlobStorage.IterateObjects()`: Azure's list API already returns each blob's name including the configured base path, but the code was building the read client by running that name through the base-path-prepending helper a second time. This doubled the base path (e.g. `gitea-lfs/gitea-lfs/aa/bb/hash`), pointing at a blob that doesn't exist. `Stat()` still succeeded because it doesn't touch the network, so the failure only surfaced when the dumper actually tried to read the object's contents. ## Solution Add `getBlobClientByFullName()`, which builds a blob client from a name that is already fully qualified, without re-applying `buildAzureBlobPath()`. `IterateObjects()` now uses it for names obtained from Azure's list API. `getBlobClient()` (used by `Open`, `Stat`, `Delete`, `ServeDirectURL`, which take relative paths) is unchanged in behavior. Also add `TestAzureBlobStorageDumpArchive`, a regression test that drives the real dump path (`IterateObjects` → `Stat` → `dump.Dumper.AddFileByReader` → `mholt/archives` zip writer) against a **non-empty** `BasePath`, and verifies the produced archive contains the object with the correct content. The existing Azure tests use an empty `BasePath` and never read object content via `IterateObjects`, which is why they didn't catch this. Fixes #35476 --------- Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
149 lines
5.0 KiB
Go
149 lines
5.0 KiB
Go
// Copyright 2023 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package storage
|
|
|
|
import (
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestObjectStoragePath(t *testing.T) {
|
|
base := ""
|
|
assert.Empty(t, buildObjectStorePath(base, "/"))
|
|
assert.Empty(t, buildObjectStorePath(base, "."))
|
|
assert.Equal(t, "a", buildObjectStorePath(base, "/a"))
|
|
assert.Equal(t, "a/b", buildObjectStorePath(base, "/a/b/"))
|
|
assert.Empty(t, buildObjectStorePathPrefix(base, ""))
|
|
assert.Equal(t, "a/", buildObjectStorePathPrefix(base, "/a/"))
|
|
|
|
base = "/"
|
|
assert.Empty(t, buildObjectStorePath(base, "/"))
|
|
assert.Empty(t, buildObjectStorePath(base, "."))
|
|
assert.Equal(t, "a", buildObjectStorePath(base, "/a"))
|
|
assert.Equal(t, "a/b", buildObjectStorePath(base, "/a/b/"))
|
|
assert.Empty(t, buildObjectStorePathPrefix(base, ""))
|
|
assert.Equal(t, "a/", buildObjectStorePathPrefix(base, "/a/"))
|
|
|
|
base = "/base"
|
|
assert.Equal(t, "base", buildObjectStorePath(base, "/"))
|
|
assert.Equal(t, "base", buildObjectStorePath(base, "."))
|
|
assert.Equal(t, "base/a", buildObjectStorePath(base, "/a"))
|
|
assert.Equal(t, "base/a/b", buildObjectStorePath(base, "/a/b/"))
|
|
assert.Equal(t, "base/", buildObjectStorePathPrefix(base, ""))
|
|
assert.Equal(t, "base/a/", buildObjectStorePathPrefix(base, "/a/"))
|
|
|
|
base = "/base/"
|
|
assert.Equal(t, "base", buildObjectStorePath(base, "/"))
|
|
assert.Equal(t, "base", buildObjectStorePath(base, "."))
|
|
assert.Equal(t, "base/a", buildObjectStorePath(base, "/a"))
|
|
assert.Equal(t, "base/a/b", buildObjectStorePath(base, "/a/b/"))
|
|
assert.Equal(t, "base/", buildObjectStorePathPrefix(base, ""))
|
|
assert.Equal(t, "base/a/", buildObjectStorePathPrefix(base, "/a/"))
|
|
}
|
|
|
|
func testStorageIterator(t *testing.T, objStore ObjectStorage) {
|
|
testFiles := [][]string{
|
|
{"a/1.txt", "a1"},
|
|
{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
|
|
{"ab/1.txt", "ab1"},
|
|
{"b/1.txt", "b1"},
|
|
{"b/2.txt", "b2"},
|
|
{"b/3.txt", "b3"},
|
|
{"b/x 4.txt", "bx4"},
|
|
}
|
|
for _, f := range testFiles {
|
|
_, err := objStore.Save(f[0], strings.NewReader(f[1]), -1)
|
|
assert.NoError(t, err)
|
|
}
|
|
defer func() {
|
|
for _, f := range testFiles {
|
|
_ = objStore.Delete(f[0])
|
|
}
|
|
}()
|
|
|
|
expectedList := map[string][]string{
|
|
"a": {"a/1.txt"},
|
|
"a/": {"a/1.txt"},
|
|
"/a/": {"a/1.txt"},
|
|
"b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
|
|
"": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
|
"/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
|
".": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
|
"a/b/../../a": {"a/1.txt"},
|
|
}
|
|
for dir, expected := range expectedList {
|
|
count := 0
|
|
err := objStore.IterateObjects(dir, func(path string, f Object) error {
|
|
content, err := io.ReadAll(f)
|
|
assert.NoError(t, err)
|
|
assert.NotEmpty(t, content)
|
|
assert.Contains(t, expected, path)
|
|
count++
|
|
return nil
|
|
})
|
|
assert.NoError(t, err)
|
|
assert.Len(t, expected, count)
|
|
}
|
|
}
|
|
|
|
func testStorageURLContentTypeAndDisposition(t *testing.T, objStore ObjectStorage) {
|
|
type expectedServeDirectHeaders struct {
|
|
ContentType string
|
|
ContentDisposition string
|
|
}
|
|
test := func(t *testing.T, s ObjectStorage, path, name string, expected expectedServeDirectHeaders, reqParams *ServeDirectOptions) {
|
|
u, err := s.ServeDirectURL(path, name, http.MethodGet, reqParams)
|
|
require.NoError(t, err)
|
|
resp, err := http.Get(u.String())
|
|
require.NoError(t, err)
|
|
defer resp.Body.Close()
|
|
if expected.ContentType != "" {
|
|
assert.Equal(t, expected.ContentType, resp.Header.Get("Content-Type"))
|
|
}
|
|
if expected.ContentDisposition != "" {
|
|
assert.Equal(t, expected.ContentDisposition, resp.Header.Get("Content-Disposition"))
|
|
}
|
|
}
|
|
|
|
testFilename := "test.txt"
|
|
_, err := objStore.Save(testFilename, strings.NewReader("dummy-content"), -1)
|
|
assert.NoError(t, err)
|
|
|
|
test(t, objStore, testFilename, "test.txt", expectedServeDirectHeaders{
|
|
ContentType: "text/plain; charset=utf-8",
|
|
ContentDisposition: `inline; filename=test.txt`,
|
|
}, nil)
|
|
|
|
test(t, objStore, testFilename, "test.pdf", expectedServeDirectHeaders{
|
|
ContentType: "application/pdf",
|
|
ContentDisposition: `inline; filename=test.pdf`,
|
|
}, nil)
|
|
|
|
test(t, objStore, testFilename, "test.wasm", expectedServeDirectHeaders{
|
|
ContentDisposition: `inline; filename=test.wasm`,
|
|
}, nil)
|
|
|
|
test(t, objStore, testFilename, "test.wasm", expectedServeDirectHeaders{
|
|
ContentType: "application/wasm",
|
|
ContentDisposition: `inline; filename=test.wasm`,
|
|
}, &ServeDirectOptions{
|
|
ContentType: "application/wasm",
|
|
})
|
|
assert.NoError(t, objStore.Delete(testFilename))
|
|
}
|
|
|
|
func testStorageGeneral(t *testing.T, objStore ObjectStorage) {
|
|
t.Run("StorageIterator", func(t *testing.T) { testStorageIterator(t, objStore) })
|
|
|
|
if _, ok := objStore.(*LocalStorage); ok {
|
|
t.Skipf("Skipping tests for local storage")
|
|
}
|
|
t.Run("StorageURLContentTypeAndDisposition", func(t *testing.T) { testStorageURLContentTypeAndDisposition(t, objStore) })
|
|
}
|