diff --git a/AGENTS.md b/AGENTS.md index 7a7034afe52..d4f25d08c82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ - In `options/locale`, only edit `locale_en-US.json`, other locales are synced automatically - In TS, use `!` instead of `?.`/`??` when a value always exists - In Go, prefer to use modern language features wherever possible +- Write sizes as multiplications like `64 * 1024`, not bit shifts like `64 << 10` - Prefer `tw-*` utilities over inline `style` and `flex-*` helpers over per-child `tw-ml-*`/`tw-mr-*` margins, falling back to `tw-*` where specificity requires `!important` - Run `make fmt` after `.go` edits, `make tidy` after `go.mod` edits, `make generate-swagger` after API changes, and lint what changed with `make lint-go`, `lint-js`, `lint-css` or `lint-templates` - Fix the cause rather than disabling a linter or weakening a test. Where unavoidable, use the narrowest scope with a trailing comment giving the reason diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index cc84922e6ab..f22034d375d 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -17,6 +17,7 @@ import ( "gitea.dev/modules/structs" "gitea.dev/modules/util" "gitea.dev/services/convert" + "gitea.dev/services/gitproxy" "gitea.dev/services/migrations" "github.com/urfave/cli/v3" @@ -94,6 +95,9 @@ func runDumpRepository(ctx context.Context, cmd *cli.Command) error { if err := git.InitSimple(); err != nil { return err } + if err := gitproxy.Run(ctx); err != nil { + return err + } log.Info("AppPath: %s", setting.AppPath) log.Info("AppWorkPath: %s", setting.AppWorkPath) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index a15b04fda50..726f18ed767 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -534,13 +534,29 @@ INTERNAL_TOKEN = ;; The value of the general Content-Security-Policy for most web pages. ;; Leave it empty to apply the default policy, or set it to "unset" to disable Content-Security-Policy. ;CONTENT_SECURITY_POLICY_GENERAL = - -;; Webhook and oauth2 clients can only call allowed hosts for security reasons. Comma separated list, eg: external, 192.168.1.0/24, *.mydomain.com -;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), * (for all hosts) -;; CIDR list: 1.2.3.0/8, 2001:db8::/32 -;; Wildcard hosts: *.mydomain.com, 192.168.100.* +;; +;; Egress mode toggles between strictness of outgoing requests: +;; Lax requires addresses to be allowed only if they are in private ranges, it allows all public ones +;; Strict requires an explicit allow of all addresses +; EGRESS_MODE = lax +;; +;; Webhook and oauth2 clients can only call allowed hosts for security reasons. Comma separated list, eg: 192.168.1.0/24:3000, [2001:db8::/32]:9090, *.mydomain.com:[80|443] +;; Built-in: loopback (for localhost), private (for LAN/intranet and CGNAT) +;; CIDR list: 1.2.3.0/8, 2001:db8::/32, and with a port [2001:db8::/32]:9090 (IPv6 addresses and CIDRs need brackets when a port is given) +;; Host matching: "example.com" matches the host and any subdomain, dot-anchored so "notexample.com" never matches; +;; "*.example.com" and ".example.com" match only subdomains, the apex itself is excluded. IDN names must be given as punycode. +;; All addresses can have ports specified. Accepted port specs: +;; a single port: 192.168.1.0/24:3000 +;; a range, both bounds inclusive: *.mydomain.com:3000-3010 +;; a bracketed set of ports and ranges, | separated: *.mydomain.com:[80|443|3000-3010] +;; all ports: *.mydomain.com:* +;; A portless entry covers all ports in Lax mode, only 80 and 443 in Strict mode +;; Port specs apply only where the list is consulted: in Lax mode that is private, loopback and CGNAT +;; targets alone, public targets are allowed on every port whatever the list says. In Strict mode every +;; target is checked, so ports restrict public hosts too. +;; Reserved addresses like link-local and cloud metadata are denied ;; This list is enforced on direct connections only. When an HTTP proxy is configured, restricting the proxied target is the proxy server's responsibility. -;ALLOWED_HOST_LIST = external +;ALLOWED_HOST_LIST = ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1808,11 +1824,13 @@ LEVEL = Info ;; Number of history information in each page ;PAGING_NUM = 10 ;; -;; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy +;; Proxy server URL, support http://, https://, socks5://, blank will follow environment http_proxy/https_proxy ;PROXY_URL = ;; ;; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. ;PROXY_HOSTS = +;; When a proxy is configured, Gitea does not enforce [security] ALLOWED_HOST_LIST on the proxied +;; target, the proxy server is expected to restrict it ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2795,20 +2813,17 @@ LEVEL = Info ;; Backoff time per http/https request retry (seconds) ;RETRY_BACKOFF = 3 ;; -;; Allowed domains for migrating, default is blank. Blank means everything will be allowed. -;; Multiple domains could be separated by commas. -;; Wildcard is supported: "github.com, *.github.com" -;ALLOWED_DOMAINS = +;; Mode toggles between strictness of scanning outgoing requests, same format as [security] EGRESS_MODE +; EGRESS_MODE = lax ;; -;; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. -;; When ALLOWED_DOMAINS is not blank, this option has a higher priority to deny domains. -;; Wildcard is supported. -;BLOCKED_DOMAINS = +;; Hosts migrations and mirrors may call, same format as [security] ALLOWED_HOST_LIST. Private and loopback addresses need a builtin or CIDR entry. +;ALLOWED_HOST_LIST = ;; -;; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default) -;; When false, migration URLs are rejected if any resolved address is private or loopback, -;; even when the host matches ALLOWED_DOMAINS: the block list is applied before the allow list. -;ALLOW_LOCALNETWORKS = false +;; Hosts migrations and mirrors may never call, portless entries defaults to all ports blocked +;BLOCKED_HOST_LIST = +;; +;; These lists are enforced on direct connections only. When a proxy is configured +;; ([proxy], [git.config] http.proxy or the environment), restricting the proxied target is the proxy server's responsibility. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -3041,10 +3056,13 @@ LEVEL = Info ;[proxy] ;; Enable the proxy, all requests to external via HTTP will be affected ;PROXY_ENABLED = false -;; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy/no_proxy +;; Proxy server URL, support http://, https://, socks5://, blank will follow environment http_proxy/https_proxy/no_proxy ;PROXY_URL = ;; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. ;PROXY_HOSTS = +;; Git remotes prefer [git.config] http.proxy and use the environment's proxy even when PROXY_ENABLED is false +;; When a proxy is configured, Gitea does not enforce the egress lists ([security], [migrations]) on the proxied +;; target, the proxy server is expected to restrict it ; [actions] ;; Enable/Disable actions capabilities diff --git a/main.go b/main.go index 0a3d0164fa2..78fe7c1e38d 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "gitea.dev/cmd" "gitea.dev/modules/log" "gitea.dev/modules/setting" + "gitea.dev/services/gitproxy" // register supported doc types _ "gitea.dev/modules/markup/console" @@ -37,6 +38,7 @@ func init() { } func main() { + gitproxy.MaybeTunnel() cli.OsExiter = func(code int) { log.GetManager().Close() os.Exit(code) diff --git a/modules/auth/openid/openid.go b/modules/auth/openid/openid.go index c0ecd6ddb5a..b2db1d3be95 100644 --- a/modules/auth/openid/openid.go +++ b/modules/auth/openid/openid.go @@ -8,9 +8,7 @@ import ( "sync" "time" - "gitea.dev/modules/hostmatcher" - "gitea.dev/modules/proxy" - "gitea.dev/modules/setting" + "gitea.dev/modules/egress" "github.com/yohcop/openid-go" ) @@ -26,15 +24,11 @@ var ( nonceStore = openid.NewSimpleNonceStore() discoveryCache = newTimedDiscoveryCache(24 * time.Hour) - // openIDInstance does discovery/verification via an SSRF-protected client, so a user-supplied - // OpenID identifier can't reach internal/loopback/reserved addresses. It honors the operator's - // [security] ALLOWED_HOST_LIST (empty defaults to "external"), matching the avatar/webhook/migration - // clients, and validates the proxy path too. Lazy: reads proxy/settings once. + // openIDInstance keeps user-supplied OpenID identifiers within [security] ALLOWED_HOST_LIST openIDInstance = sync.OnceValue(func() *openid.OpenID { - allowList := hostmatcher.ParseHostMatchList("security.ALLOWED_HOST_LIST", setting.Security.AllowedHostList) return openid.NewOpenID(&http.Client{ Timeout: 30 * time.Second, - Transport: hostmatcher.NewHTTPTransport("openid", allowList, nil, proxy.Proxy(), setting.Proxy.ProxyURLFixed, nil), + Transport: egress.NewSecurityPolicy("openid").NewHTTPTransport(), }) }) ) diff --git a/modules/egress/policies.go b/modules/egress/policies.go new file mode 100644 index 00000000000..317ded5bf1f --- /dev/null +++ b/modules/egress/policies.go @@ -0,0 +1,117 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package egress + +import ( + "cmp" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "os" + "strings" + + "gitea.dev/modules/egress/policy" + "gitea.dev/modules/proxy" + "gitea.dev/modules/setting" + + "golang.org/x/net/http/httpproxy" +) + +func NewMigrationPolicy() *policy.Policy { + return newMigrationPolicy(proxy.Proxy()) +} + +// NewGitPolicy is the migration policy for the git proxy, which keeps git's own proxy choice +func NewGitPolicy() (*policy.Policy, error) { + selectProxy, err := gitProxySelector() + if err != nil { + return nil, err + } + return newMigrationPolicy(selectProxy), nil +} + +// gitProxySelector picks proxies like git did: [git.config] http.proxy, else a [proxy] PROXY_URL, else the environment incl. ALL_PROXY +func gitProxySelector() (func(*http.Request) (*url.URL, error), error) { + env := httpproxy.FromEnvironment() + if rawURL, ok := setting.GitConfig.Options["http.proxy"]; ok { + gitProxy, err := normalizeGitProxy(rawURL) + if err != nil { + return nil, fmt.Errorf("[git.config] http.proxy: %w", err) + } + env.HTTPProxy, env.HTTPSProxy = gitProxy, gitProxy + return requestProxy(env), nil + } + if setting.Proxy.Enabled && setting.Proxy.ProxyURL != "" { + return proxy.Proxy(), nil + } + allProxy := cmp.Or(os.Getenv("all_proxy"), os.Getenv("ALL_PROXY")) + env.HTTPProxy, env.HTTPSProxy = cmp.Or(env.HTTPProxy, allProxy), cmp.Or(env.HTTPSProxy, allProxy) + return requestProxy(env), nil +} + +func requestProxy(cfg *httpproxy.Config) func(*http.Request) (*url.URL, error) { + proxyFunc := cfg.ProxyFunc() + return func(req *http.Request) (*url.URL, error) { return proxyFunc(req.URL) } +} + +// normalizeGitProxy reads a proxy URL the way git reads http.proxy: http is the default scheme, 1080 curl's default port +func normalizeGitProxy(rawURL string) (string, error) { + if rawURL == "" { + return "", nil + } + if !strings.Contains(rawURL, "://") { + rawURL = "http://" + rawURL + } + proxyURL, err := url.Parse(rawURL) + if err != nil { + return "", errors.New("invalid URL") // the parse error would echo its credentials + } + if proxyURL.Scheme == "http" && proxyURL.Port() == "" { + proxyURL.Host = net.JoinHostPort(proxyURL.Hostname(), "1080") + } + return proxyURL.String(), nil +} + +func newMigrationPolicy(selectProxy func(*http.Request) (*url.URL, error)) *policy.Policy { + return policy.NewPolicy("migrations", policyMode(setting.Migrations.EgressMode), + policy.WithAllow(setting.Migrations.AllowedHostList, "migrations.ALLOWED_HOST_LIST"), + policy.WithBlock(setting.Migrations.BlockedHostList, "migrations.BLOCKED_HOST_LIST"), + policy.WithLocalNeedsIPAllow(), + policy.WithProxy(selectProxy)) +} + +func NewWebhookPolicy() *policy.Policy { + var p *policy.Policy + selectProxy := proxy.WebHookProxy() + if webhookProxy := setting.Webhook.ProxyURLFixed; webhookProxy != nil { + next := selectProxy + selectProxy = func(req *http.Request) (*url.URL, error) { + u, err := next(req) + if err == nil && u == webhookProxy { + err = p.CheckHost(req.URL) // the webhook proxy resolves the target, so only its name can be checked + } + return u, err + } + } + p = policy.NewPolicy("webhook", policyMode(setting.Security.EgressMode), + policy.WithAllow(setting.Webhook.AllowedHostList, "security.ALLOWED_HOST_LIST"), + policy.WithProxy(selectProxy)) + + return p +} + +func NewSecurityPolicy(usage string) *policy.Policy { + return policy.NewPolicy(usage, policyMode(setting.Security.EgressMode), + policy.WithAllow(setting.Security.AllowedHostList, "security.ALLOWED_HOST_LIST"), + policy.WithProxy(proxy.Proxy())) +} + +func policyMode(mode string) policy.Mode { + if mode == "strict" { + return policy.Strict + } + return policy.Lax +} diff --git a/modules/egress/policies_test.go b/modules/egress/policies_test.go new file mode 100644 index 00000000000..fb5d8f76b15 --- /dev/null +++ b/modules/egress/policies_test.go @@ -0,0 +1,100 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package egress + +import ( + "net/http" + "net/url" + "testing" + + "gitea.dev/modules/setting" + "gitea.dev/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + require.NoError(t, err) + return u +} + +func TestNewMigrationPolicy(t *testing.T) { + defer test.MockVariableValue(&setting.Migrations)() + for _, tc := range []struct { + allow, block, mode, target string + want bool + }{ + {target: "https://1.2.3.4", want: true}, + {allow: "github.com", target: "https://10.0.0.1"}, // a hostname allow doesn't cover a private IP + {allow: "github.com", target: "https://8.8.8.8", want: true}, // lax exempts public targets + {allow: "github.com", mode: "strict", target: "https://8.8.8.8"}, + {block: "8.8.0.0/16", target: "https://8.8.8.8"}, + } { + setting.Migrations.AllowedHostList, setting.Migrations.BlockedHostList, setting.Migrations.EgressMode = tc.allow, tc.block, tc.mode + u, err := url.Parse(tc.target) + require.NoError(t, err) + err = NewMigrationPolicy().CheckHostIPs(u) + assert.Equal(t, tc.want, err == nil, "%+v: %v", tc, err) + } +} + +func TestWebhookPolicyProxy(t *testing.T) { + proxyURL := &url.URL{Scheme: "http", Host: "localhost:8080"} + defer test.MockVariableValue(&setting.Security.EgressMode, "lax")() + defer test.MockVariableValue(&setting.Webhook.AllowedHostList, "discordapp.com,s.discordapp.com")() + defer test.MockVariableValue(&setting.Webhook.ProxyURL, proxyURL.String())() + defer test.MockVariableValue(&setting.Webhook.ProxyURLFixed, proxyURL)() + defer test.MockVariableValue(&setting.Webhook.ProxyHosts, []string{"*.discordapp.com", "discordapp.com"})() + selectProxy := NewWebhookPolicy().NewHTTPTransport().Proxy + + for target, want := range map[string]string{ + "https://discordapp.com/api/webhooks/xxxxxxxxx/xxxxxxxxxxxxxxxxxxx": proxyURL.String(), + "http://s.discordapp.com/assets/xxxxxx": proxyURL.String(), + "http://github.com/a/b": "", + "http://www.discordapp.com/assets/xxxxxx": proxyURL.String(), + } { + req, err := http.NewRequest(http.MethodPost, target, nil) + require.NoError(t, err) + req.Host = "" + u, err := selectProxy(req) + require.NoError(t, err, target) + if want == "" { + assert.Nil(t, u, target) + } else { + assert.Equal(t, want, u.String(), target) + } + } +} + +func TestSecurityPolicy(t *testing.T) { + defer test.MockVariableValue(&setting.Security.AllowedHostList, "avatars.example.com")() + defer test.MockVariableValue(&setting.Security.EgressMode, "lax")() + lax := NewSecurityPolicy("test") + assert.NoError(t, lax.CheckHost(mustURL(t, "https://avatars.example.com"))) + assert.NoError(t, lax.CheckHost(mustURL(t, "https://8.8.8.8"))) // lax exempts public targets + assert.Error(t, lax.CheckHost(mustURL(t, "https://10.0.0.1"))) // restricted targets still need an allow entry + + setting.Security.EgressMode = "strict" + strict := NewSecurityPolicy("test") + assert.NoError(t, strict.CheckHost(mustURL(t, "https://avatars.example.com"))) + assert.Error(t, strict.CheckHost(mustURL(t, "https://8.8.8.8"))) +} + +func TestNewGitPolicy(t *testing.T) { + gitConfig := map[string]string{"http.proxy": "proxy.corp"} + defer test.MockVariableValue(&setting.GitConfig.Options, gitConfig)() + gitPolicy, err := NewGitPolicy() + require.NoError(t, err) + selected, err := gitPolicy.Proxy(&http.Request{URL: &url.URL{Scheme: "https", Host: "git.example.com"}}) + require.NoError(t, err) + assert.Equal(t, "http://proxy.corp:1080", selected.String()) + + gitConfig["http.proxy"] = "http://user:secret@[::1" + _, err = NewGitPolicy() + require.Error(t, err) + assert.NotContains(t, err.Error(), "secret") +} diff --git a/modules/egress/policy/matchlist.go b/modules/egress/policy/matchlist.go new file mode 100644 index 00000000000..1bd3077ce05 --- /dev/null +++ b/modules/egress/policy/matchlist.go @@ -0,0 +1,460 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package policy + +import ( + "errors" + "fmt" + "net" + "net/netip" + "slices" + "strconv" + "strings" + "sync" + "unicode/utf8" +) + +// matchList keeps a list of IPs and hostnames +type matchList struct { + patterns []domainRule + ipv6List, ipv4List []prefixRule + rejected []string +} + +type portRange struct { + start uint16 + end uint16 +} + +func (p *portRange) Contains(port uint16) bool { + return port >= p.start && port <= p.end +} + +type prefixRule struct { + prefix netip.Prefix + portRanges []portRange +} + +func (p *prefixRule) Contains(port netip.AddrPort) bool { + return p.prefix.Contains(port.Addr()) && slices.ContainsFunc(p.portRanges, func(r portRange) bool { + return r.Contains(port.Port()) + }) +} + +type domainRule struct { + pattern string + portRanges []portRange +} + +func (p *domainRule) Contains(hostname string, port uint16) bool { + return matchDomain(p.pattern, hostname) && slices.ContainsFunc(p.portRanges, func(r portRange) bool { + return r.Contains(port) + }) +} + +const ( + AliasPrivate = "private" + AliasLoopback = "loopback" +) + +var namedRanges = sync.OnceValue(func() map[string][]netip.Prefix { + base := map[string][]string{ + // private ranges and CGNAT + AliasPrivate: {"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7", "100.64.0.0/10"}, + AliasLoopback: {"127.0.0.0/8", "::1/128"}, + } + out := make(map[string][]netip.Prefix, 2) + for name, ranges := range base { + out[name] = make([]netip.Prefix, 0, len(ranges)) + for _, r := range ranges { + out[name] = append(out[name], netip.MustParsePrefix(r)) + } + } + return out +}) + +type ( + AllowList struct{ matchList } + BlockList struct{ matchList } +) + +type listEntry interface { + addTo(*matchList) +} + +func (p prefixRule) addTo(m *matchList) { + if p.prefix.Addr().Is4() { + m.ipv4List = append(m.ipv4List, p) + } else { + m.ipv6List = append(m.ipv6List, p) + } +} + +func (p domainRule) addTo(m *matchList) { m.patterns = append(m.patterns, p) } + +type aliasExpansion []prefixRule + +func (r aliasExpansion) addTo(m *matchList) { + for _, pr := range r { + pr.addTo(m) + } +} + +func parseList(hostlist string, mode Mode, isBlocklist bool) (list matchList) { + for entry := range strings.SplitSeq(hostlist, ",") { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + rule, err := parseRule(entry, mode, isBlocklist) + if err != nil { + list.rejected = append(list.rejected, err.Error()) + continue + } + rule.addTo(&list) + } + return list +} + +func splitEntry(entry string) (target, portSpec string, err error) { + if strings.HasPrefix(entry, "[") { + end := strings.IndexByte(entry, ']') + if end < 0 { + return "", "", errors.New("missing closing bracket") + } + target = entry[1:end] + if target == "" { + return "", "", errors.New("empty host") + } + rest := entry[end+1:] + if rest == "" { + return target, "", nil + } + if !strings.HasPrefix(rest, ":") { + return "", "", fmt.Errorf("unexpected %q after bracketed target", rest) + } + portSpec = rest[1:] + if portSpec == "" { + return "", "", errors.New("empty port") + } + return target, portSpec, nil + } + + // check for host:port + if strings.Count(entry, ":") == 1 { + i := strings.IndexByte(entry, ':') + target, portSpec = entry[:i], entry[i+1:] + switch { + case target == "": + return "", "", fmt.Errorf("empty host: '%s'", entry) + case portSpec == "": + return "", "", fmt.Errorf("empty port: '%s'", entry) + } + return target, portSpec, nil + } + + // Portless: patterns, IPs, CIDRs - and multi-colon forms + return entry, "", nil +} + +func parseRule(entry string, mode Mode, isBlocklist bool) (listEntry, error) { + target, portSpec, err := splitEntry(entry) + if err != nil { + return nil, fmt.Errorf("failed to split entry %s: %w", entry, err) + } + portRanges, err := parsePortSpec(portSpec, mode, isBlocklist) + if err != nil { + return nil, fmt.Errorf("invalid port syntax on entry %s: %w", entry, err) + } + return classifyTarget(target, portRanges) +} + +func classifyTarget(target string, ranges []portRange) (listEntry, error) { + target = strings.ToLower(strings.TrimSpace(target)) + target = strings.TrimSuffix(target, ".") + + if expanded, ok := newNamedRanges(target, ranges); ok { + return expanded, nil + } + if prefix, err := newPrefixRule(target, ranges); !errors.Is(err, errNotIP) { + return prefix, err + } + return newDomainRule(target, ranges) +} + +var errNotIP = errors.New("not an IP address") + +func newPrefixRule(target string, ranges []portRange) (prefixRule, error) { + if strings.ContainsRune(target, '/') { + prefix, err := netip.ParsePrefix(target) + if err != nil { + if strings.ContainsRune(target, ':') { + return prefixRule{}, fmt.Errorf("invalid IPv6 CIDR %q (unbracketed IPv6 with port? use [addr] or [addr]:port): %w", target, err) + } + return prefixRule{}, fmt.Errorf("invalid CIDR %q: %w", target, err) + } + if prefix.Bits() == 0 { + return prefixRule{}, fmt.Errorf("catch-all CIDR %q covers every address and is not allowed", target) + } + if masked := prefix.Masked(); masked != prefix { + return prefixRule{}, fmt.Errorf("invalid CIDR %q: host bits must be zero, use %q", target, masked) + } + return prefixRule{prefix: prefix, portRanges: ranges}, nil + } + + addr, addrErr := netip.ParseAddr(target) + if addrErr == nil { + if addr.Zone() != "" { + return prefixRule{}, fmt.Errorf("invalid address %q: address zone is not dialable", target) + } + addr = addr.Unmap() + return prefixRule{prefix: netip.PrefixFrom(addr, addr.BitLen()), portRanges: ranges}, nil + } + + if !ipShaped(target) { + return prefixRule{}, errNotIP + } + return prefixRule{}, fmt.Errorf("target %q looks like an IP address but is not a valid one: %w", target, addrErr) +} + +func ipShaped(target string) bool { + if strings.ContainsRune(target, ':') { + return true + } + return strings.ContainsRune(target, '.') && !strings.ContainsFunc(target, func(r rune) bool { + return (r < '0' || r > '9') && r != '.' + }) +} + +// newNamedRanges expands a named range alias into one prefixRule per CIDR. ok +// is false when target is not an alias. +func newNamedRanges(target string, ranges []portRange) (aliasExpansion, bool) { + prefixes, ok := namedRanges()[target] + if !ok { + return nil, false + } + expanded := make(aliasExpansion, len(prefixes)) + for i, prefix := range prefixes { + expanded[i] = prefixRule{prefix: prefix, portRanges: ranges} + } + return expanded, true +} + +// newDomainRule classifies a hostname pattern. +func newDomainRule(target string, ranges []portRange) (domainRule, error) { + if target == "*" { + return domainRule{}, fmt.Errorf("catch-all host pattern %q matches every host and is not allowed", target) + } + if target == "external" { + return domainRule{}, errors.New(`the "external" builtin was replaced by EGRESS_MODE = lax`) + } + if !validDomainPattern(target) { + return domainRule{}, fmt.Errorf("target %q is not an IP, CIDR, named range, or valid hostname pattern", target) + } + return domainRule{pattern: target, portRanges: ranges}, nil +} + +// parsePortSpec parses a port spec: "" means the context default per defaultPorts, "*" +// all ports, otherwise a port, a "lo-hi" range, or a bracketed "[p|p-p|...]" set. +func parsePortSpec(spec string, mode Mode, isBlocklist bool) ([]portRange, error) { + if spec == "" { + return defaultPorts(mode, isBlocklist), nil + } + if spec == "*" { + return []portRange{{start: 0, end: 65535}}, nil + } + if strings.HasPrefix(spec, "[") && strings.HasSuffix(spec, "]") { + inner := spec[1 : len(spec)-1] + if inner == "" { + return nil, fmt.Errorf("empty port set %q", spec) + } + var ranges []portRange + for item := range strings.SplitSeq(inner, "|") { + r, err := parsePortItem(item) + if err != nil { + return nil, err + } + ranges = append(ranges, r) + } + return ranges, nil + } + r, err := parsePortItem(spec) + if err != nil { + return nil, err + } + return []portRange{r}, nil +} + +// parsePortItem parses "p" or "lo-hi" with 1 <= lo <= hi <= 65535. The parts +// are trimmed, so a bracketed set may be spaced ("[80 | 443]"). +func parsePortItem(item string) (portRange, error) { + lo, hi, isRange := strings.Cut(item, "-") + start, err := parsePort(strings.TrimSpace(lo)) + if err != nil { + return portRange{}, err + } + end := start + if isRange { + end, err = parsePort(strings.TrimSpace(hi)) + if err != nil { + return portRange{}, err + } + } + if start > end { + return portRange{}, fmt.Errorf("reversed port range %q", item) + } + return portRange{start: start, end: end}, nil +} + +func parsePort(s string) (uint16, error) { + p, err := strconv.ParseUint(s, 10, 16) + if err != nil || p == 0 { + return 0, fmt.Errorf("invalid port %q", s) + } + return uint16(p), nil +} + +// defaultPorts: a portless block entry covers every port, a portless allow entry covers every port in Lax mode and only the web ports in Strict. +func defaultPorts(mode Mode, isBlocklist bool) []portRange { + if isBlocklist || mode == Lax { + return []portRange{{start: 0, end: 65535}} + } + return []portRange{{start: 80, end: 80}, {start: 443, end: 443}} +} + +func NewAllowList(hostList string, mode Mode) *AllowList { + return &AllowList{parseList(hostList, mode, false)} +} + +func NewBlockList(hostList string) *BlockList { + return &BlockList{parseList(hostList, Strict, true)} +} + +func (m *matchList) MatchHostname(host string, port uint16) bool { + host = strings.ToLower(strings.TrimSpace(host)) + if hostname, _, err := net.SplitHostPort(host); err == nil { + host = hostname + } + host = strings.TrimSuffix(host, ".") + + for _, pattern := range m.patterns { + if pattern.Contains(host, port) { + return true + } + } + return false +} + +// matchDomain implements the domain-matching model of x/net/http/httpproxy +// (dot-anchored suffix), like curl's and Go's NO_PROXY: +// - "example.com" matches that host and any subdomain, dot-anchored so +// "notexample.com" never matches +// - "*.example.com" and ".example.com" match only subdomains, apex +// excluded — the two spellings are equivalent, mirroring httpproxy's normalization +func matchDomain(pattern, host string) bool { + if strings.HasPrefix(pattern, "*.") || strings.HasPrefix(pattern, ".") { + suffix := strings.TrimPrefix(pattern, "*") + return strings.HasSuffix(host, suffix) && len(host) > len(suffix) + } + return pattern == host || strings.HasSuffix(host, "."+pattern) +} + +// validDomainPattern reports whether p is a usable domain pattern: any glob +// metacharacter beyond the documented forms (?, character classes, backslash +// escapes, mid-pattern *) is rejected instead of silently never matching, and +// so is any non-ASCII rune, which byte-wise matching against a resolved +// hostname could never hit (IDN names must be given as punycode). +func validDomainPattern(p string) bool { + if strings.HasPrefix(p, "*.") || strings.HasPrefix(p, ".") { + p = p[1:] + } + if p == "" || strings.ContainsFunc(p, func(r rune) bool { return r >= utf8.RuneSelf }) { + return false + } + return !strings.ContainsAny(p, " *?[]/:\\") +} + +// Rejected returns the entries dropped at parse, so callers can log them at startup. +func (m *matchList) Rejected() []string { + return slices.Clone(m.rejected) +} + +// MatchIPAddr checks if the given IP is in the list. +func (m *matchList) MatchIPAddr(ip netip.AddrPort) bool { + match := m.ipv4List + if ip.Addr().Is6() { + match = m.ipv6List + } + for _, prefix := range match { + if prefix.Contains(ip) { + return true + } + } + return false +} + +func (m *matchList) IsEmpty() bool { + return len(m.patterns) == 0 && len(m.ipv4List) == 0 && len(m.ipv6List) == 0 +} + +type addrClass uint8 + +const ( + classPublic addrClass = iota + classRestricted + classReserved +) + +var cgnatRange = netip.MustParsePrefix("100.64.0.0/10") // RFC 6598 + +// reservedRanges are never dialable, based on https://microsoft.github.io/AntiSSRF/ipaddressranges.html +var reservedRanges = func() (ranges []netip.Prefix) { + for _, cidr := range []string{ + "0.0.0.0/8", // "this network" + "100.100.100.200/32", // Alibaba Cloud metadata + "168.63.129.16/32", // Azure WireServer + "169.254.0.0/16", // link-local, cloud metadata endpoints + "192.0.0.0/24", // IETF protocol assignments + "192.0.2.0/24", // TEST-NET-1 + "192.31.196.0/24", // AS112 + "192.52.193.0/24", // AMT + "192.88.99.0/24", // 6to4 relay anycast + "192.175.48.0/24", // AS112 + "198.18.0.0/15", // benchmarking + "198.51.100.0/24", // TEST-NET-2 + "203.0.113.0/24", // TEST-NET-3 + "224.0.0.0/4", // multicast + "240.0.0.0/4", // reserved, incl. limited broadcast + "::/96", // IPv4-compatible, embeds IPv4 + "::ffff:0:0:0/96", // IPv4-translated, embeds IPv4 + "64:ff9b::/96", // wkp NAT64 + "64:ff9b:1::/48", // local-use NAT64 + "100::/64", // discard-only + "100:0:0:1::/64", // dummy + "2001::/23", // IETF protocol assignments, incl. Teredo and ORCHID + "2001:db8::/32", // documentation + "2002::/16", // 6to4, embeds IPv4 + "2620:4f:8000::/48", // AS112 + "3fff::/20", // documentation + "5f00::/16", // SRv6 SIDs + "fd00:ec2::254/128", // AWS IMDS + "fe80::/10", // link-local + "fec0::/10", // site-local + "ff00::/8", // multicast + } { + ranges = append(ranges, netip.MustParsePrefix(cidr)) + } + return ranges +}() + +// classifyAddr reports the class of a canonical address. +func classifyAddr(ip netip.Addr) addrClass { + switch { + case ip.Zone() != "" || !ip.IsLoopback() && slices.ContainsFunc(reservedRanges, func(p netip.Prefix) bool { return p.Contains(ip) }): + return classReserved + case ip.IsPrivate() || ip.IsLoopback() || cgnatRange.Contains(ip): + return classRestricted + } + return classPublic +} diff --git a/modules/egress/policy/matchlist_test.go b/modules/egress/policy/matchlist_test.go new file mode 100644 index 00000000000..386956599db --- /dev/null +++ b/modules/egress/policy/matchlist_test.go @@ -0,0 +1,134 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package policy + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMatchHostname(t *testing.T) { + for _, tc := range []struct { + pattern, host string + port uint16 + want bool + }{ + // a bare entry matches its host and subdomains like NO_PROXY, a wildcard or leading dot matches subdomains only + {pattern: "example.com", host: "example.com", port: 80, want: true}, + {pattern: "example.com", host: "sub.example.com", port: 80, want: true}, + {pattern: "example.com", host: "notexample.com", port: 80}, // dot-anchored + {pattern: "*.example.com", host: "sub.example.com", port: 80, want: true}, + {pattern: ".example.com", host: "sub.example.com", port: 80, want: true}, + {pattern: "*.example.com", host: "example.com", port: 80}, // apex never matches + {pattern: "*.example.com", host: "notexample.com", port: 80}, + {pattern: ".example.com", host: "notexample.com", port: 80}, + // matching is case-insensitive and tolerates spaces, a port suffix and a trailing dot + {pattern: "example.com", host: " EXAMPLE.com.:8080 ", port: 80, want: true}, + // strict entries without a port cover the web ports 80 and 443 + {pattern: "example.com", host: "example.com", port: 443, want: true}, + {pattern: "example.com", host: "example.com", port: 8080}, + {pattern: "example.com:*", host: "example.com", port: 8080, want: true}, + {pattern: "example.com:8080", host: "example.com", port: 8080, want: true}, + {pattern: "example.com:8080", host: "example.com", port: 80}, + {pattern: "example.com:[80|443-445]", host: "example.com", port: 444, want: true}, + {pattern: "example.com:[80|443-445]", host: "example.com", port: 446}, + } { + assert.Equalf(t, tc.want, NewAllowList(tc.pattern, Strict).MatchHostname(tc.host, tc.port), "pattern %q host %q port %d", tc.pattern, tc.host, tc.port) + } + // lax entries without a port cover every port + assert.True(t, NewAllowList("example.com", Lax).MatchHostname("example.com", 8080)) + + assert.True(t, NewAllowList(" , ", Strict).IsEmpty(), "blank entries are skipped") + assert.True(t, NewAllowList("", Strict).IsEmpty()) + assert.False(t, NewAllowList("example.com", Strict).IsEmpty()) +} + +func TestMatchIPAddr(t *testing.T) { + for _, tc := range []struct { + pattern, ip string + port uint16 + want bool + }{ + {pattern: "10.0.0.0/8", ip: "10.1.2.3", port: 80, want: true}, + {pattern: "10.0.0.0/8", ip: "11.1.2.3", port: 80}, + // a bare IP entry carries the default ports like a hostname entry + {pattern: "192.168.1.1", ip: "192.168.1.1", port: 443, want: true}, + {pattern: "192.168.1.1", ip: "192.168.1.1", port: 8080}, + {pattern: "10.0.0.0/8:22", ip: "10.1.2.3", port: 22, want: true}, + {pattern: "10.0.0.0/8:22", ip: "10.1.2.3", port: 80}, + {pattern: "2001:db8::/64", ip: "2001:db8::1", port: 443, want: true}, + {pattern: "2001:db8::/64", ip: "2001:db8::1", port: 8080}, + {pattern: "[2001:db8::/64]:9418", ip: "2001:db8::1", port: 9418, want: true}, + {pattern: "[2001:db8::/64]:9418", ip: "2001:db9::1", port: 9418}, + {pattern: "[::1]:8080", ip: "::1", port: 8080, want: true}, + {pattern: "[::1]:8080", ip: "::1", port: 80}, + // named ranges expand to CIDRs + {pattern: "loopback", ip: "127.0.0.1", port: 80, want: true}, + {pattern: "loopback", ip: "::1", port: 80, want: true}, + {pattern: "loopback", ip: "10.1.2.3", port: 80}, + {pattern: "private", ip: "100.64.0.1", port: 80, want: true}, // CGNAT + {pattern: "private", ip: "8.8.8.8", port: 80}, + {pattern: "private:22", ip: "fd00::1", port: 22, want: true}, + {pattern: "private:22", ip: "fd00::1", port: 80}, + } { + addr := netip.AddrPortFrom(netip.MustParseAddr(tc.ip), tc.port) + assert.Equalf(t, tc.want, NewAllowList(tc.pattern, Strict).MatchIPAddr(addr), "pattern %q ip %s port %d", tc.pattern, tc.ip, tc.port) + } +} + +func TestBlockListDefaultPorts(t *testing.T) { + // deny entries without a port cover every port, allow entries default to the web ports in strict mode + assert.True(t, NewBlockList("example.com").MatchHostname("example.com", 22)) + assert.True(t, NewBlockList("10.0.0.0/8").MatchIPAddr(netip.AddrPortFrom(netip.MustParseAddr("10.0.0.5"), 12345))) + + blocked := NewBlockList("example.com:22") + assert.True(t, blocked.MatchHostname("example.com", 22)) + assert.False(t, blocked.MatchHostname("example.com", 80)) +} + +func TestRejectedEntries(t *testing.T) { + for _, tc := range []struct{ entry, wantErr string }{ + {entry: "*", wantErr: `catch-all host pattern "*" matches every host and is not allowed`}, + {entry: "external", wantErr: "EGRESS_MODE"}, + {entry: "0.0.0.0/0", wantErr: `catch-all CIDR "0.0.0.0/0" covers every address and is not allowed`}, + {entry: "::/0", wantErr: `catch-all CIDR "::/0" covers every address and is not allowed`}, + {entry: "10.0.0.5/8", wantErr: `host bits must be zero, use "10.0.0.0/8"`}, + {entry: "fe80::/64:80", wantErr: "unbracketed IPv6 with port"}, + {entry: "[2001:db8::]/64", wantErr: `unexpected "/64" after bracketed target`}, + {entry: "fe80::1%eth0", wantErr: "address zone is not dialable"}, + {entry: "999.1.1.1", wantErr: `target "999.1.1.1" looks like an IP address but is not a valid one`}, + {entry: "sub.*.example.com", wantErr: "is not an IP, CIDR, named range, or valid hostname pattern"}, + {entry: "exämple.com", wantErr: "is not an IP, CIDR, named range, or valid hostname pattern"}, + {entry: "example.com:99999", wantErr: `invalid port "99999"`}, + {entry: "example.com:443-80", wantErr: `reversed port range "443-80"`}, + {entry: "example.com:[]", wantErr: `empty port set "[]"`}, + {entry: ":80", wantErr: "empty host: ':80'"}, + {entry: "example.com:", wantErr: "empty port: 'example.com:'"}, + {entry: "[::1", wantErr: "missing closing bracket"}, + {entry: "[::1]:", wantErr: "empty port"}, + {entry: "[]:80", wantErr: "empty host"}, + } { + list := NewAllowList(tc.entry, Strict) + require.Lenf(t, list.Rejected(), 1, "entry %q", tc.entry) + assert.Containsf(t, list.Rejected()[0], tc.wantErr, "entry %q", tc.entry) + } + + // the deny list rejects the same catch-alls, a rejected entry matches nothing + for _, entry := range []string{"*", "0.0.0.0/0", "::/0"} { + block := NewBlockList(entry) + require.Lenf(t, block.Rejected(), 1, "block entry %q", entry) + assert.Truef(t, block.IsEmpty(), "block entry %q", entry) + } + + // a rejected entry drops only itself + list := NewAllowList("example.com, 10.0.0.5/8, example.org", Strict) + assert.True(t, list.MatchHostname("example.com", 80)) + assert.True(t, list.MatchHostname("example.org", 443)) + require.Len(t, list.Rejected(), 1) + assert.Contains(t, list.Rejected()[0], `use "10.0.0.0/8"`) + assert.Empty(t, NewAllowList("example.com", Strict).Rejected()) +} diff --git a/modules/egress/policy/policy.go b/modules/egress/policy/policy.go new file mode 100644 index 00000000000..e5f74f1b02b --- /dev/null +++ b/modules/egress/policy/policy.go @@ -0,0 +1,252 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package policy + +import ( + "cmp" + "context" + "errors" + "fmt" + "net" + "net/http" + "net/netip" + "net/url" + "strconv" + "sync" + "syscall" + "time" +) + +type Mode uint8 + +const ( + // Lax mode allows public and unresolved targets by default, restricted ones need an allow entry + Lax Mode = iota + // Strict mode requires every target to match an allow entry + Strict +) + +// ErrDenied wraps a policy rejection, so callers can tell it from a network failure. +var ErrDenied = errors.New("denied by egress policy") + +type Policy struct { + usage string + mode Mode + allow AllowList + block BlockList + allowKey, blockKey string // the settings the lists were read from, named in rejections + localNeedsIPAllow bool + proxyFunc func(*http.Request) (*url.URL, error) + proxyAddrs sync.Map // dial addresses proxyFunc returned, the operator's proxies are exempt from the lists +} + +type Option func(*Policy) + +// WithAllow sets the allow list from the setting named by key +func WithAllow(hostList, key string) Option { + return func(p *Policy) { + p.allow, p.allowKey = *NewAllowList(hostList, p.mode), key + } +} + +func WithBlock(hostList, key string) Option { + return func(p *Policy) { + p.block, p.blockKey = *NewBlockList(hostList), key + } +} + +// WithLocalNeedsIPAllow requires private, loopback and CGNAT targets to match an IP allow entry (CIDR or named range), a host name match is not enough. +func WithLocalNeedsIPAllow() Option { + return func(p *Policy) { + p.localNeedsIPAllow = true + } +} + +func WithProxy(proxyFunc func(*http.Request) (*url.URL, error)) Option { + return func(p *Policy) { + p.proxyFunc = proxyFunc + } +} + +// NewPolicy compiles a policy enforced on every outbound dial, usage names the caller in rejections. +// The mode sets the policy posture and the default ports of portless allow entries. +func NewPolicy(usage string, mode Mode, opts ...Option) *Policy { + p := &Policy{usage: usage, mode: mode} // the zero lists are valid and behave as empty ones + for _, opt := range opts { + opt(p) + } + return p +} + +// proxyPorts maps the proxy schemes net/http speaks to their default ports, it speaks HTTP to any other +var proxyPorts = map[string]string{"http": "80", "https": "443", "socks5": "1080", "socks5h": "1080"} + +// targetPorts maps target schemes to their default dial port, http and anything else dials 80 +var targetPorts = map[string]uint16{"https": 443, "git": 9418} + +// dialPort resolves the port a target URL is dialed on, empty port means the scheme default +func dialPort(u *url.URL) uint16 { + if port, err := strconv.ParseUint(u.Port(), 10, 16); err == nil && port != 0 { + return uint16(port) + } + return cmp.Or(targetPorts[u.Scheme], 80) +} + +// ProxyDialAddr returns the address the transport dials for proxy URL u +func ProxyDialAddr(u *url.URL) string { + return net.JoinHostPort(u.Hostname(), cmp.Or(u.Port(), proxyPorts[u.Scheme])) +} + +func (p *Policy) blockedError(target string) error { + return fmt.Errorf("%s can not call blocked HTTP servers (check your %s setting), deny '%s'", p.usage, p.blockKey, target) +} + +func (p *Policy) notAllowedError(target string) error { + return fmt.Errorf("%s can only call allowed HTTP servers (check your %s setting), deny '%s'", p.usage, p.allowKey, target) +} + +func (p *Policy) checkAddr(host string, ip netip.AddrPort) error { + ip = netip.AddrPortFrom(ip.Addr().Unmap(), ip.Port()) + class := classifyAddr(ip.Addr()) + if class == classReserved { + return fmt.Errorf("%s can not call reserved addresses, deny '%s'", p.usage, denyTarget(host, ip)) + } + return p.gate(host, ip, class) +} + +// gate enforces the deny list, then the allow list, lax mode exempts public and unresolved targets, the dial-time check classifies the resolved address +func (p *Policy) gate(host string, ip netip.AddrPort, class addrClass) error { + if err := p.blockReason(host, ip); err != nil { + return err + } + if p.mode == Lax && (class == classPublic) { + return nil + } + return p.allowCheck(host, ip, class) +} + +// allowCheck returns nil when the allow list names the target, else an error naming the allow entry it needs +func (p *Policy) allowCheck(host string, ip netip.AddrPort, class addrClass) error { + if p.allow.MatchIPAddr(ip) { + return nil + } + // with localNeedsIPAllow a restricted target needs an IP entry, a hostname match is not enough + hostnameOk := !p.localNeedsIPAllow || class != classRestricted + if hostnameOk && p.allow.MatchHostname(host, ip.Port()) { + return nil + } + if p.mode == Strict { + return p.notAllowedError(denyTarget(host, ip)) + } + if !hostnameOk { + return fmt.Errorf("%s needs an explicit IP allow entry (private/loopback/CGNAT)", denyTarget(host, ip)) + } + return fmt.Errorf("%s needs an explicit allow entry (private/loopback/CGNAT)", denyTarget(host, ip)) +} + +func (p *Policy) blockReason(host string, ip netip.AddrPort) error { + if p.block.MatchHostname(host, ip.Port()) || p.block.MatchIPAddr(ip) { + return p.blockedError(denyTarget(host, ip)) + } + return nil +} + +// denyTarget renders the checked address for denial messages, host is empty for an IP literal, ip is invalid for an unresolved host name +func denyTarget(host string, ip netip.AddrPort) string { + if !ip.Addr().IsValid() { + return host + } + if host == "" { + return ip.Addr().String() + } + return fmt.Sprintf("%s(%s)", host, ip.Addr()) +} + +// CheckHost pre-screens a target URL whose host name may be unresolved or an IP literal. +func (p *Policy) CheckHost(u *url.URL) error { + host, port := u.Hostname(), dialPort(u) + ip, err := netip.ParseAddr(host) + addrPort := netip.AddrPortFrom(ip, port) + if err == nil { + return p.checkAddr("", addrPort) + } + return p.checkAddr(host, addrPort) // invalid ip doesn't match anything +} + +// CheckHostIPs reports whether u's host may be called, it resolves the host and every address must pass as the dialer may pick any. +func (p *Policy) CheckHostIPs(u *url.URL) error { + // hosts behind a proxy may have no DNS resolver, the name-only CheckHost screen still applies + ips, _ := net.LookupIP(u.Hostname()) + return p.checkHostIPs(u, ips) +} + +func (p *Policy) checkHostIPs(u *url.URL, ips []net.IP) error { + if len(ips) == 0 { + return p.CheckHost(u) + } + host, port := u.Hostname(), dialPort(u) + for _, ip := range ips { + addr, _ := netip.AddrFromSlice(ip) + addrPort := netip.AddrPortFrom(addr, port) + if err := p.checkAddr(host, addrPort); err != nil { + return err + } + } + return nil +} + +// NewDialContext returns a dial function that checks the resolved address at connect time, so DNS rebinding can't bypass it. +func (p *Policy) NewDialContext() func(ctx context.Context, network, addr string) (net.Conn, error) { + return p.dialContext(false) +} + +// dialContext can let through the proxies the selector returned, for a transport dialing proxies and targets alike +func (p *Policy) dialContext(allowProxies bool) func(ctx context.Context, network, addr string) (net.Conn, error) { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + dialer := net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} + if _, isProxy := p.proxyAddrs.Load(addr); !allowProxies || !isProxy { + host, _, err := net.SplitHostPort(addr) // the requested host, also on redirects where the request's Host is empty + if err != nil { + return nil, err + } + dialer.Control = func(_, ipAddr string, _ syscall.RawConn) error { + addrPort, err := netip.ParseAddrPort(ipAddr) + if err != nil { + return fmt.Errorf("%s can only call HTTP servers via TCP, deny '%s(%s)': %w", p.usage, host, ipAddr, err) + } + if err := p.checkAddr(host, addrPort); err != nil { + return fmt.Errorf("%w: %w", ErrDenied, err) + } + return nil + } + } + return dialer.DialContext(ctx, network, addr) + } +} + +// Proxy selects the proxy for req and lets the dialer reach it. +func (p *Policy) Proxy(req *http.Request) (proxyURL *url.URL, err error) { + if p.proxyFunc != nil { + proxyURL, err = p.proxyFunc(req) + } + if proxyURL != nil { + if _, ok := proxyPorts[proxyURL.Scheme]; !ok { + return nil, fmt.Errorf("unsupported proxy scheme %q, use http, https or socks5", proxyURL.Scheme) + } + p.proxyAddrs.LoadOrStore(ProxyDialAddr(proxyURL), struct{}{}) + } + return proxyURL, err +} + +func (p *Policy) NewHTTPTransport() *http.Transport { + return &http.Transport{ + Proxy: p.Proxy, + DialContext: p.dialContext(true), + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } +} diff --git a/modules/egress/policy/policy_test.go b/modules/egress/policy/policy_test.go new file mode 100644 index 00000000000..3d1b413a2d8 --- /dev/null +++ b/modules/egress/policy/policy_test.go @@ -0,0 +1,160 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package policy + +import ( + "net" + "net/http" + "net/netip" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckAddr(t *testing.T) { + for _, tc := range []struct { + name, allow, block, host, ip string + localNeedsIPAllow, strict, want bool + }{ + // lax mode exempts public and unresolved targets, restricted ones need an allow entry + {name: "empty lists allow public", host: "github.com", ip: "8.8.8.8", want: true}, + {name: "empty lists deny private", ip: "10.0.0.5"}, + {name: "lax exempts public despite allow list", allow: "example.com", host: "other.com", ip: "8.8.8.8", want: true}, + {name: "lax exempts unresolved despite allow list", allow: "example.com", host: "other.com", want: true}, + {name: "block unresolved", block: "evil.example.com", host: "evil.example.com"}, + {name: "block host", block: "evil.example.com", host: "evil.example.com", ip: "8.8.8.8"}, + {name: "block cidr", block: "127.0.0.0/8", ip: "127.0.0.1"}, + {name: "block ipv4-mapped", block: "loopback", ip: "::ffff:127.0.0.1"}, + {name: "block nat64 by cidr", block: "10.0.0.0/8", ip: "64:ff9b::a00:1"}, + {name: "allow loopback", allow: "loopback", ip: "127.0.0.1", want: true}, + {name: "allow host", allow: "example.com", host: "example.com", ip: "8.8.8.8", want: true}, + {name: "allow cidr", allow: "10.0.0.0/8", ip: "10.0.0.5", want: true}, + {name: "block overrides allow", allow: "10.0.0.0/8", block: "10.0.0.5/32", ip: "10.0.0.5"}, + {name: "reserved denied by cidr", allow: "169.254.0.0/16", ip: "169.254.169.254"}, + {name: "reserved denied ipv4-mapped", allow: "169.254.0.0/16", ip: "::ffff:169.254.169.254"}, + {name: "local gate ignores host", allow: "example.com", host: "example.com", ip: "10.0.0.5", localNeedsIPAllow: true}, + {name: "local gate accepts builtin", allow: "private", ip: "100.64.0.1", localNeedsIPAllow: true, want: true}, + {name: "local gate accepts cidr", allow: "10.0.0.0/24", ip: "10.0.0.5", localNeedsIPAllow: true, want: true}, + // strict mode requires every target to match the allow list, an empty list denies all + {name: "strict denies public with empty list", ip: "8.8.8.8", strict: true}, + {name: "strict denies unresolved with empty list", host: "example.com", strict: true}, + {name: "strict allows unresolved host", allow: "example.com", host: "example.com", strict: true, want: true}, + {name: "strict rejects unlisted public", allow: "loopback", ip: "8.8.8.8", strict: true}, + {name: "strict rejects unmatched host", allow: "example.com", host: "other.com", ip: "8.8.8.8", strict: true}, + {name: "strict allows matched host", allow: "example.com", host: "example.com", ip: "8.8.8.8", strict: true, want: true}, + {name: "strict block overrides allow", allow: "10.0.0.0/8", block: "10.0.0.5/32", ip: "10.0.0.5", strict: true}, + {name: "strict reserved denied by cidr", allow: "169.254.0.0/16", ip: "169.254.169.254", strict: true}, + {name: "strict local gate ignores host", allow: "example.com", host: "example.com", ip: "10.0.0.5", localNeedsIPAllow: true, strict: true}, + {name: "strict local gate accepts builtin", allow: "private", ip: "100.64.0.1", localNeedsIPAllow: true, strict: true, want: true}, + } { + opts := []Option{WithAllow(tc.allow, "test.ALLOWED"), WithBlock(tc.block, "test.BLOCKED")} + if tc.localNeedsIPAllow { + opts = append(opts, WithLocalNeedsIPAllow()) + } + var addr netip.Addr + if tc.ip != "" { + addr = netip.MustParseAddr(tc.ip) + } + mode := Lax + if tc.strict { + mode = Strict + } + err := NewPolicy("test", mode, opts...).checkAddr(tc.host, netip.AddrPortFrom(addr, 80)) + assert.Equal(t, tc.want, err == nil, "%s: %v", tc.name, err) + } +} + +// a policy without list options behaves like one with empty lists +func TestPolicyWithoutLists(t *testing.T) { + lax := NewPolicy("test", Lax) + assert.NoError(t, lax.checkAddr("", netip.AddrPortFrom(netip.MustParseAddr("8.8.8.8"), 80))) // public targets pass + assert.Error(t, lax.checkAddr("", netip.AddrPortFrom(netip.MustParseAddr("10.0.0.5"), 80))) // restricted targets need an entry + strict := NewPolicy("test", Strict) + err := strict.checkAddr("", netip.AddrPortFrom(netip.MustParseAddr("8.8.8.8"), 80)) // strict denies without an entry + assert.ErrorContains(t, err, "can only call allowed HTTP servers") +} + +func TestDenialNamesSetting(t *testing.T) { + err := NewPolicy("webhook", Strict, WithAllow("example.com", "security.ALLOWED_HOST_LIST")).checkAddr("other.com", netip.AddrPortFrom(netip.MustParseAddr("8.8.8.8"), 80)) + assert.EqualError(t, err, "webhook can only call allowed HTTP servers (check your security.ALLOWED_HOST_LIST setting), deny 'other.com(8.8.8.8)'") + + err = NewPolicy("webhook", Lax, WithBlock("evil.com", "migrations.BLOCKED_HOST_LIST")).CheckHost(hostURL(t, "http://evil.com")) + assert.EqualError(t, err, "webhook can not call blocked HTTP servers (check your migrations.BLOCKED_HOST_LIST setting), deny 'evil.com'") +} + +func hostURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + require.NoError(t, err) + return u +} + +func TestCheckHostIPs(t *testing.T) { + ips := func(addrs ...string) (ret []net.IP) { + for _, addr := range addrs { + ret = append(ret, net.ParseIP(addr)) + } + return ret + } + + blocked := NewPolicy("test", Lax, WithAllow("private", ""), WithBlock("blocked.example.com", "")) + assert.NoError(t, blocked.checkHostIPs(hostURL(t, "http://example.com"), ips("8.8.8.8", "10.0.0.5"))) + assert.NoError(t, blocked.checkHostIPs(hostURL(t, "http://example.com"), nil)) // unresolved name, the dialer re-checks the resolved address + assert.Error(t, blocked.checkHostIPs(hostURL(t, "http://blocked.example.com"), ips("8.8.8.8"))) + assert.Error(t, blocked.checkHostIPs(hostURL(t, "http://blocked.example.com"), nil)) + + allowed := NewPolicy("test", Strict, WithAllow("10.0.0.0/8, *.example.com", "")) + assert.NoError(t, allowed.checkHostIPs(hostURL(t, "http://"), ips("10.0.0.5"))) + assert.NoError(t, allowed.checkHostIPs(hostURL(t, "http://git.example.com"), ips("192.168.0.1"))) + assert.NoError(t, allowed.checkHostIPs(hostURL(t, "http://git.example.com"), nil)) + assert.Error(t, allowed.checkHostIPs(hostURL(t, "http://other.com"), ips("10.0.0.5", "192.168.0.1"))) + assert.Error(t, allowed.checkHostIPs(hostURL(t, "http://other.com"), nil)) + + builtins := NewPolicy("test", Lax, WithAllow("private, loopback", "")) + assert.NoError(t, builtins.checkHostIPs(hostURL(t, "http://example.com"), ips("8.8.8.8", "100.64.0.1", "::1"))) + for _, ip := range []string{ + "0.1.2.3", "100.100.100.200", "168.63.129.16", "169.254.169.254", "192.0.2.1", "192.88.99.1", "198.18.0.1", + "198.51.100.1", "203.0.113.1", "::7f00:1", "::ffff:0:a00:5", "64:ff9b::a9fe:a9fe", "64:ff9b::808:808", "2001::1", "2001:db8::1", + "2002::1", "fe80::1", + } { + assert.Error(t, builtins.checkHostIPs(hostURL(t, "http://example.com"), ips(ip)), ip) + } +} + +func TestNewDialContext(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + addr := ln.Addr().String() + + dial := func(proxy string, allowProxies bool) error { + policy := NewPolicy("test", Lax, WithBlock("loopback", ""), WithProxy(http.ProxyURL(&url.URL{Scheme: "http", Host: proxy}))) + _, _ = policy.Proxy(&http.Request{}) + conn, err := policy.dialContext(allowProxies)(t.Context(), "tcp", addr) + if err == nil { + _ = conn.Close() + } + return err + } + assert.NoError(t, dial(addr, true)) + assert.ErrorIs(t, dial(addr, false), ErrDenied) + assert.ErrorIs(t, dial("127.0.0.1:1", true), ErrDenied) +} + +func TestProxy(t *testing.T) { + for raw, want := range map[string]string{ + "http://127.0.0.1": "127.0.0.1:80", + "socks5://[::1]": "[::1]:1080", + "https://proxy.corp:8443": "proxy.corp:8443", + } { + u, err := url.Parse(raw) + require.NoError(t, err) + assert.Equal(t, want, ProxyDialAddr(u), raw) + } + + _, err := NewPolicy("test", Lax, WithProxy(http.ProxyURL(&url.URL{Scheme: "socks4", Host: "proxy.corp:1080"}))).Proxy(&http.Request{}) + assert.ErrorContains(t, err, "unsupported proxy scheme") +} diff --git a/modules/git/gitcmd/command.go b/modules/git/gitcmd/command.go index 81e375cabf9..e8ec12ebc6c 100644 --- a/modules/git/gitcmd/command.go +++ b/modules/git/gitcmd/command.go @@ -255,10 +255,14 @@ func commonBaseEnvs() []string { // CommonGitCmdEnvs returns the common environment variables for a "git" command. func CommonGitCmdEnvs() []string { - return append(commonBaseEnvs(), []string{ + envs := append(commonBaseEnvs(), []string{ "LC_ALL=C", // ensure git output is in English, error messages are parsed in English "GIT_TERMINAL_PROMPT=0", // avoid prompting for credentials interactively, supported since git v2.3 }...) + if extra := extraEnvs.Load(); extra != nil { + envs = append(envs, *extra...) + } + return envs } // CommonCmdServEnvs is like CommonGitCmdEnvs, but it only returns minimal required environment variables for the "gitea serv" command diff --git a/modules/git/gitcmd/env.go b/modules/git/gitcmd/env.go index 4f8b860e372..dc71aadb288 100644 --- a/modules/git/gitcmd/env.go +++ b/modules/git/gitcmd/env.go @@ -6,6 +6,7 @@ package gitcmd import ( "fmt" "os/exec" + "sync/atomic" "gitea.dev/modules/log" "gitea.dev/modules/setting" @@ -13,6 +14,13 @@ import ( var GitExecutable = "git" // the command name of git, will be updated to an absolute path during initialization +var extraEnvs atomic.Pointer[[]string] + +// SetExtraEnvs adds envs to every git command, the git proxy routes git's network remotes with them +func SetExtraEnvs(envs []string) { + extraEnvs.Store(&envs) +} + // SetExecutablePath changes the path of git executable and checks the file permission and version. func SetExecutablePath(path string) error { // If path is empty, we use the default value of GitExecutable "git" to search for the location of git. diff --git a/modules/git/redirection.go b/modules/git/redirection.go deleted file mode 100644 index ee7a365482d..00000000000 --- a/modules/git/redirection.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2026 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import "gitea.dev/modules/git/gitcmd" - -func HandleGitCmdHTTPRedirection(cmd *gitcmd.Command, targets ...string) { - // Protect from SSRF vector (e.g. migrating from an attacker URL). - // cmd.AddConfig("http.followRedirects", "false") - // However, we can't do so at the moment: - // this fails due to 301: git -c http.followRedirects=false clone -v https://gitlab.com/{owner}/{repo} - // this succeeds: git -c http.followRedirects=false clone -v https://gitlab.com/{owner}/{repo}.git - // FIXME: GIT-CLONE-HTTP-REDIRECT-SSRF: need a complete solution in the future -} diff --git a/modules/git/repo.go b/modules/git/repo.go index 302dc4d5ca1..bb21e8c2389 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -7,7 +7,6 @@ package git import ( "context" "fmt" - "net/url" "os" "path" "path/filepath" @@ -19,7 +18,6 @@ import ( "gitea.dev/modules/cache" "gitea.dev/modules/git/gitcmd" "gitea.dev/modules/git/gitrepo" - "gitea.dev/modules/proxy" "gitea.dev/modules/setting" "gitea.dev/modules/util" ) @@ -183,7 +181,6 @@ func Clone(ctx context.Context, from, to string, opts CloneRepoOptions) error { } cmd := gitcmd.NewCommand().AddArguments("clone") - HandleGitCmdHTTPRedirection(cmd, from, to) if opts.SkipTLSVerify { cmd.AddArguments("-c", "http.sslVerify=false") } @@ -220,20 +217,7 @@ func Clone(ctx context.Context, from, to string, opts CloneRepoOptions) error { opts.Timeout = -1 } - envs := os.Environ() - if opts.Env != nil { - envs = opts.Env - } else { - u, err := url.Parse(from) - if err == nil { - envs = proxy.EnvWithProxy(u) - } - } - - return cmd. - WithTimeout(opts.Timeout). - WithEnv(envs). - RunWithStderr(ctx) + return cmd.WithTimeout(opts.Timeout).WithEnv(opts.Env).RunWithStderr(ctx) } // PushOptions options when push to remote diff --git a/modules/git/repo_test.go b/modules/git/repo_test.go index 9a4dc932cda..aea3bec3ab5 100644 --- a/modules/git/repo_test.go +++ b/modules/git/repo_test.go @@ -4,10 +4,7 @@ package git import ( - "net/http" - "net/http/httptest" "path/filepath" - "sync/atomic" "testing" "github.com/stretchr/testify/assert" @@ -22,24 +19,3 @@ func TestRepoIsEmpty(t *testing.T) { assert.NoError(t, err) assert.True(t, isEmpty) } - -// TestCloneRefusesRedirects ensures Clone never follows HTTP redirects, so a remote -// cannot redirect to an otherwise-blocked address (SSRF, e.g. during migration). -func TestCloneRefusesRedirects(t *testing.T) { - t.Skip("FIXME: GIT-CLONE-HTTP-REDIRECT-SSRF: need a complete solution in the future") - var targetHit atomic.Bool - target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - targetHit.Store(true) - w.WriteHeader(http.StatusNotFound) - })) - defer target.Close() - - redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound) - })) - defer redirect.Close() - - err := Clone(t.Context(), redirect.URL, filepath.Join(t.TempDir(), "dst"), CloneRepoOptions{}) - assert.Error(t, err) - assert.False(t, targetHit.Load(), "git must not follow the redirect to the target") -} diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go deleted file mode 100644 index c82a57b7b98..00000000000 --- a/modules/hostmatcher/hostmatcher.go +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package hostmatcher - -import ( - "net" - "path/filepath" - "slices" - "strings" - "sync" -) - -// HostMatchList is used to check if a host or IP is in a list. -type HostMatchList struct { - SettingKeyHint string - SettingValue string - - // builtins networks - builtins []string - // patterns for host names (with wildcard support) - patterns []string - // ipNets is the CIDR network list - ipNets []*net.IPNet -} - -// MatchBuiltinExternal A valid global-unicast IP that is neither private (see MatchBuiltinPrivate) -// nor a reserved special-purpose range (see reservedIPNets); i.e. a routable host on the public internet. -const MatchBuiltinExternal = "external" - -// reservedIPNets are special-purpose ranges that net.IP.IsPrivate omits but that must not be -// treated as public/external destinations (CGNAT, cloud metadata, IPv6 transition, etc.). We layer -// these on top of net.IP.IsPrivate (RFC 1918 / RFC 4193) so future additions to Go's IsPrivate are -// picked up automatically, while still covering the ranges it leaves out; otherwise the default -// allow-list would let authenticated users reach cloud metadata, internal, and IPv6 transition -// endpoints (SSRF), and a "private" block-list would fail to catch them. -var reservedIPNets = sync.OnceValue(func() []*net.IPNet { - var nets []*net.IPNet - for _, cidr := range []string{ - // IPv4 - "100.64.0.0/10", // RFC 6598 Carrier-Grade NAT - "168.63.129.16/32", // Azure WireServer metadata endpoint - "192.0.0.0/24", // RFC 6890 IETF protocol assignments - "192.0.2.0/24", // RFC 5737 TEST-NET-1 - "192.88.99.0/24", // RFC 7526 6to4 relay anycast (deprecated) - "198.18.0.0/15", // RFC 2544 benchmarking - "198.51.100.0/24", // RFC 5737 TEST-NET-2 - "203.0.113.0/24", // RFC 5737 TEST-NET-3 - // IPv6 - "100::/64", // RFC 6666 discard-only - "64:ff9b::/96", // RFC 6052 NAT64 (can embed IPv4 such as 169.254.169.254) - "64:ff9b:1::/48", // RFC 8215 local-use NAT64 - "2001::/32", // RFC 4380 Teredo tunneling (embeds IPv4) - "2001:10::/28", // RFC 4843 ORCHID (deprecated) - "2001:20::/28", // RFC 7343 ORCHIDv2 - "2001:db8::/32", // RFC 3849 documentation - "2002::/16", // RFC 3056 6to4 (embeds IPv4) - } { - _, ipNet, err := net.ParseCIDR(cidr) - if err != nil { - panic("hostmatcher: invalid reserved CIDR " + cidr + ": " + err.Error()) - } - nets = append(nets, ipNet) - } - return nets -}) - -// isReservedIP reports whether ip falls in reserved special-purpose -// range (see reservedIPNets) that must not be considered a public/external destination. -func isReservedIP(ip net.IP) bool { - for _, ipNet := range reservedIPNets() { - if ipNet.Contains(ip) { - return true - } - } - return false -} - -// MatchBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7), -// plus the reserved special-purpose ranges in reservedIPNets (CGNAT, NAT64, cloud metadata, etc.). -// Also called LAN/Intranet. -const MatchBuiltinPrivate = "private" - -// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. -const MatchBuiltinLoopback = "loopback" - -func isBuiltin(s string) bool { - return s == MatchBuiltinExternal || s == MatchBuiltinPrivate || s == MatchBuiltinLoopback -} - -// ParseHostMatchList parses the host list HostMatchList -func ParseHostMatchList(settingKeyHint, hostList string) *HostMatchList { - hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList} - for s := range strings.SplitSeq(hostList, ",") { - s = strings.ToLower(strings.TrimSpace(s)) - if s == "" { - continue - } - _, ipNet, err := net.ParseCIDR(s) - if err == nil { - hl.ipNets = append(hl.ipNets, ipNet) - } else if isBuiltin(s) { - hl.builtins = append(hl.builtins, s) - } else { - hl.patterns = append(hl.patterns, s) - } - } - return hl -} - -// ParseSimpleMatchList parse a simple matchlist (no built-in networks, no CIDR support, only wildcard pattern match) -func ParseSimpleMatchList(settingKeyHint, matchList string) *HostMatchList { - hl := &HostMatchList{ - SettingKeyHint: settingKeyHint, - SettingValue: matchList, - } - for s := range strings.SplitSeq(matchList, ",") { - s = strings.ToLower(strings.TrimSpace(s)) - if s == "" { - continue - } - // we keep the same result as old `matchlist`, so no builtin/CIDR support here, we only match wildcard patterns - hl.patterns = append(hl.patterns, s) - } - return hl -} - -// AppendBuiltin appends more builtins to match -func (hl *HostMatchList) AppendBuiltin(builtin string) { - hl.builtins = append(hl.builtins, builtin) -} - -// IsEmpty checks if the checklist is empty -func (hl *HostMatchList) IsEmpty() bool { - return hl == nil || (len(hl.builtins) == 0 && len(hl.patterns) == 0 && len(hl.ipNets) == 0) -} - -func (hl *HostMatchList) checkPattern(host string) bool { - host = strings.ToLower(strings.TrimSpace(host)) - for _, pattern := range hl.patterns { - if matched, _ := filepath.Match(pattern, host); matched { - return true - } - } - return false -} - -// matchesIP determines if the given IP matches any of the configured rules -func (hl *HostMatchList) matchesIP(ip net.IP) bool { - if slices.Contains(hl.patterns, "*") { - return true - } - for _, builtin := range hl.builtins { - switch builtin { - case MatchBuiltinExternal: - // External address must be a global unicast, must not be in reserved range and must not be in private range - if ip.IsGlobalUnicast() && !isReservedIP(ip) && !ip.IsPrivate() { - return true - } - case MatchBuiltinPrivate: - // Private address must be global unicast, must not be in range we explicitly exclude for security reasons - // and must be in private range - if ip.IsGlobalUnicast() && !isReservedIP(ip) && ip.IsPrivate() { - return true - } - case MatchBuiltinLoopback: - if ip.IsLoopback() { - return true - } - } - } - for _, ipNet := range hl.ipNets { - if ipNet.Contains(ip) { - return true - } - } - return false -} - -// MatchHostName checks if the host matches an allow/deny(block) list -func (hl *HostMatchList) MatchHostName(host string) bool { - if hl == nil { - return false - } - - hostname, _, err := net.SplitHostPort(host) - if err != nil { - hostname = host - } - if hl.checkPattern(hostname) { - return true - } - if ip := net.ParseIP(hostname); ip != nil { - return hl.matchesIP(ip) - } - return false -} - -// MatchIPAddr checks if the IP matches an allow/deny(block) list, it's safe to pass `nil` to `ip` -func (hl *HostMatchList) MatchIPAddr(ip net.IP) bool { - if hl == nil { - return false - } - host := ip.String() // nil-safe, we will get "" if ip is nil - return hl.checkPattern(host) || hl.matchesIP(ip) -} - -// MatchHostOrIP checks if the host or IP matches an allow/deny(block) list -func (hl *HostMatchList) MatchHostOrIP(host string, ip net.IP) bool { - return hl.MatchHostName(host) || hl.MatchIPAddr(ip) -} diff --git a/modules/hostmatcher/hostmatcher_test.go b/modules/hostmatcher/hostmatcher_test.go deleted file mode 100644 index 464354ff411..00000000000 --- a/modules/hostmatcher/hostmatcher_test.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package hostmatcher - -import ( - "net" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestHostOrIPMatchesList(t *testing.T) { - type tc struct { - host string - ip net.IP - expected bool - } - - // for IPv6: "::1" is loopback, "fd00::/8" is private - - hl := ParseHostMatchList("", "private, External, *.myDomain.com, 169.254.1.0/24") - - test := func(cases []tc) { - for _, c := range cases { - assert.Equalf(t, c.expected, hl.MatchHostOrIP(c.host, c.ip), "case domain=%s, ip=%v, expected=%v", c.host, c.ip, c.expected) - } - } - - cases := []tc{ - {"", net.IPv4zero, false}, - {"", net.IPv6zero, false}, - - {"", net.ParseIP("127.0.0.1"), false}, - {"127.0.0.1", nil, false}, - {"", net.ParseIP("::1"), false}, - - {"", net.ParseIP("10.0.1.1"), true}, - {"10.0.1.1", nil, true}, - {"10.0.1.1:8080", nil, true}, - {"", net.ParseIP("192.168.1.1"), true}, - {"192.168.1.1", nil, true}, - {"", net.ParseIP("fd00::1"), true}, - {"fd00::1", nil, true}, - - {"", net.ParseIP("8.8.8.8"), true}, - {"", net.ParseIP("1001::1"), true}, - - {"mydomain.com", net.IPv4zero, false}, - {"sub.mydomain.com", net.IPv4zero, true}, - {"sub.mydomain.com:8080", net.IPv4zero, true}, - - {"", net.ParseIP("169.254.1.1"), true}, - {"169.254.1.1", nil, true}, - {"", net.ParseIP("169.254.2.2"), false}, - {"169.254.2.2", nil, false}, - } - test(cases) - - hl = ParseHostMatchList("", "loopback") - cases = []tc{ - {"", net.IPv4zero, false}, - {"", net.ParseIP("127.0.0.1"), true}, - {"", net.ParseIP("10.0.1.1"), false}, - {"", net.ParseIP("192.168.1.1"), false}, - {"", net.ParseIP("8.8.8.8"), false}, - - {"", net.ParseIP("::1"), true}, - {"", net.ParseIP("fd00::1"), false}, - {"", net.ParseIP("1000::1"), false}, - - {"mydomain.com", net.IPv4zero, false}, - } - test(cases) - - hl = ParseHostMatchList("", "private") - cases = []tc{ - {"", net.IPv4zero, false}, - {"", net.ParseIP("127.0.0.1"), false}, - {"", net.ParseIP("10.0.1.1"), true}, - {"", net.ParseIP("192.168.1.1"), true}, - {"", net.ParseIP("8.8.8.8"), false}, - - {"", net.ParseIP("::1"), false}, - {"", net.ParseIP("fd00::1"), true}, - {"", net.ParseIP("1000::1"), false}, - - {"mydomain.com", net.IPv4zero, false}, - } - test(cases) - - hl = ParseHostMatchList("", "external") - cases = []tc{ - {"", net.IPv4zero, false}, - {"", net.ParseIP("127.0.0.1"), false}, - {"", net.ParseIP("10.0.1.1"), false}, - {"", net.ParseIP("192.168.1.1"), false}, - {"", net.ParseIP("8.8.8.8"), true}, - - {"", net.ParseIP("::1"), false}, - {"", net.ParseIP("fd00::1"), false}, - {"", net.ParseIP("1000::1"), true}, - - {"mydomain.com", net.IPv4zero, false}, - } - test(cases) - - hl = ParseHostMatchList("", "*") - cases = []tc{ - {"", net.IPv4zero, true}, - {"", net.ParseIP("127.0.0.1"), true}, - {"", net.ParseIP("10.0.1.1"), true}, - {"", net.ParseIP("192.168.1.1"), true}, - {"", net.ParseIP("8.8.8.8"), true}, - - {"", net.ParseIP("::1"), true}, - {"", net.ParseIP("fd00::1"), true}, - {"", net.ParseIP("1000::1"), true}, - - {"mydomain.com", net.IPv4zero, true}, - } - test(cases) - - // built-in network names can be escaped (warping the first char with `[]`) to be used as a real host name - // this mechanism is reversed for internal usage only (maybe for some rare cases), it's not supposed to be used by end users - // a real user should never use loopback/private/external as their host names - hl = ParseHostMatchList("", "loopback, [p]rivate") - cases = []tc{ - {"loopback", nil, false}, - {"", net.ParseIP("127.0.0.1"), true}, - {"private", nil, true}, - {"", net.ParseIP("192.168.1.1"), false}, - } - test(cases) - - hl = ParseSimpleMatchList("", "loopback, *.domain.com") - cases = []tc{ - {"loopback", nil, true}, - {"", net.ParseIP("127.0.0.1"), false}, - {"sub.domain.com", nil, true}, - {"other.com", nil, false}, - {"", net.ParseIP("1.1.1.1"), false}, - } - test(cases) - - hl = ParseSimpleMatchList("", "external") - cases = []tc{ - {"", net.ParseIP("192.168.1.1"), false}, - {"", net.ParseIP("1.1.1.1"), false}, - {"external", nil, true}, - } - test(cases) - - hl = ParseSimpleMatchList("", "") - cases = []tc{ - {"", net.ParseIP("192.168.1.1"), false}, - {"", net.ParseIP("1.1.1.1"), false}, - {"external", nil, false}, - } - test(cases) -} - -// TestReservedRanges ensures special-purpose ranges that net.IP.IsPrivate misses are kept out of the -// "external" allow-list (the default for webhook delivery and repository migrations) and folded into -// the "private" block-list, so they cannot be used for SSRF to metadata/internal endpoints. -func TestReservedRanges(t *testing.T) { - external := ParseHostMatchList("", "external") - private := ParseHostMatchList("", "private") - - // legitimate public destinations: external, not private - for _, ip := range []string{"8.8.8.8", "1.1.1.1", "2001:4860:4860::8888", "1000::1"} { - addr := net.ParseIP(ip) - assert.Truef(t, external.MatchIPAddr(addr), "public ip %s should be external", ip) - assert.Falsef(t, private.MatchIPAddr(addr), "public ip %s should not be private", ip) - } - - // RFC 1918 / RFC 4193 private ranges (now folded into privateIPNets instead of net.IP.IsPrivate): - // not external, blockable as private. Includes range edges to guard the CIDR boundaries. - for _, ip := range []string{ - "10.0.0.0", "10.255.255.255", // 10.0.0.0/8 - "172.16.0.0", "172.31.255.255", // 172.16.0.0/12 - "192.168.0.0", "192.168.255.255", // 192.168.0.0/16 - "fc00::", "fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", // fc00::/7 - } { - addr := net.ParseIP(ip) - assert.Falsef(t, external.MatchIPAddr(addr), "private ip %s must not be external", ip) - assert.Truef(t, private.MatchIPAddr(addr), "private ip %s should match private block-list", ip) - } - - // 172.32.0.0 is just outside 172.16.0.0/12: a public destination, not private - if addr := net.ParseIP("172.32.0.0"); assert.NotNil(t, addr) { - assert.True(t, external.MatchIPAddr(addr), "172.32.0.0 should be external") - assert.False(t, private.MatchIPAddr(addr), "172.32.0.0 should not be private") - } - - // reserved ranges that IsPrivate does not cover: not external, but blockable as private - for _, ip := range []string{ - "100.64.0.1", // CGNAT - "100.127.255.254", // CGNAT - "168.63.129.16", // Azure WireServer - "192.0.2.1", // TEST-NET-1 - "198.18.0.1", // benchmarking - "198.51.100.1", // TEST-NET-2 - "203.0.113.1", // TEST-NET-3 - "169.254.169.254", // Cloud metadata - "192.88.99.1", // 6to4 relay anycast - "64:ff9b::1", // NAT64 - "64:ff9b::a9fe:a9fe", // NAT64 embedding 169.254.169.254 - "2001::1", // Teredo - "2002::1", // 6to4 - "2001:db8::1", // documentation - "fe80::1", // link local address - } { - addr := net.ParseIP(ip) - assert.Falsef(t, external.MatchIPAddr(addr), "reserved ip %s must not be external", ip) - assert.Falsef(t, private.MatchIPAddr(addr), "reserved ip %s should match private block-list", ip) - } -} diff --git a/modules/hostmatcher/http.go b/modules/hostmatcher/http.go deleted file mode 100644 index 1ee9893c24a..00000000000 --- a/modules/hostmatcher/http.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package hostmatcher - -import ( - "context" - "crypto/tls" - "fmt" - "net" - "net/http" - "net/url" - "syscall" - "time" -) - -// NewDialContext returns a DialContext for Transport, the DialContext will do allow/block list check -func NewDialContext(usage string, allowList, blockList *HostMatchList, proxy *url.URL) func(ctx context.Context, network, addr string) (net.Conn, error) { - // How Go HTTP Client works with redirection: - // transport.RoundTrip URL=http://domain.com, Host=domain.com - // transport.DialContext addrOrHost=domain.com:80 - // dialer.Control tcp4:11.22.33.44:80 - // transport.RoundTrip URL=http://www.domain.com/, Host=(empty here, in the direction, HTTP client doesn't fill the Host field) - // transport.DialContext addrOrHost=domain.com:80 - // dialer.Control tcp4:11.22.33.44:80 - return func(ctx context.Context, network, addrOrHost string) (net.Conn, error) { - dialer := net.Dialer{ - // default values comes from http.DefaultTransport - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - - Control: func(network, ipAddr string, c syscall.RawConn) error { - host, port, err := net.SplitHostPort(addrOrHost) - if err != nil { - return err - } - if proxy != nil { - // Always allow the host of the proxy, but only on the specified port. - if host == proxy.Hostname() && port == proxy.Port() { - return nil - } - } - - // in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here - tcpAddr, err := net.ResolveTCPAddr(network, ipAddr) - if err != nil { - return fmt.Errorf("%s can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%w", usage, host, network, ipAddr, err) - } - - var blockedError error - if blockList.MatchHostOrIP(host, tcpAddr.IP) { - blockedError = fmt.Errorf("%s can not call blocked HTTP servers (check your %s setting), deny '%s(%s)'", usage, blockList.SettingKeyHint, host, ipAddr) - } - - // if we have an allow-list, check the allow-list first - if !allowList.IsEmpty() { - if !allowList.MatchHostOrIP(host, tcpAddr.IP) { - return fmt.Errorf("%s can only call allowed HTTP servers (check your %s setting), deny '%s(%s)'", usage, allowList.SettingKeyHint, host, ipAddr) - } - } - // otherwise, we always follow the blocked list - return blockedError - }, - } - return dialer.DialContext(ctx, network, addrOrHost) - } -} - -// NewHTTPTransport builds an http.Transport that validates the request target against the allow/block -// lists on the direct-dial path (DialContext). When an HTTP proxy is configured the proxy resolves and -// dials the target itself, so restricting the proxied target is the proxy server's responsibility, not -// Gitea's. proxyFunc selects the proxy URL per request (the http.Transport.Proxy selector, e.g. -// proxy.Proxy()); proxyURLFixed is the fixed proxy address the dialer must always permit; tlsConfig may -// be nil. blockList may be nil for callers that only maintain an allow-list. -func NewHTTPTransport(usage string, allowList, blockList *HostMatchList, proxyFunc func(*http.Request) (*url.URL, error), proxyURLFixed *url.URL, tlsConfig *tls.Config) *http.Transport { - return &http.Transport{ - TLSClientConfig: tlsConfig, - Proxy: proxyFunc, - DialContext: NewDialContext(usage, allowList, blockList, proxyURLFixed), - } -} diff --git a/modules/proxy/proxy.go b/modules/proxy/proxy.go index 38e323494d9..981811b59e0 100644 --- a/modules/proxy/proxy.go +++ b/modules/proxy/proxy.go @@ -6,8 +6,6 @@ package proxy import ( "net/http" "net/url" - "os" - "strings" "sync" "gitea.dev/modules/glob" @@ -16,82 +14,48 @@ import ( ) var ( - once sync.Once - hostMatchers []glob.Glob + globalProxyHosts = sync.OnceValue(func() []glob.Glob { return compileHosts(setting.Proxy.ProxyHosts) }) + webhookProxyHosts = sync.OnceValue(func() []glob.Glob { return compileHosts(setting.Webhook.ProxyHosts) }) ) -// GetProxyURL returns proxy url -func GetProxyURL() string { - if !setting.Proxy.Enabled { - return "" - } - - if setting.Proxy.ProxyURL == "" { - if os.Getenv("http_proxy") != "" { - return os.Getenv("http_proxy") +func compileHosts(hosts []string) (globs []glob.Glob) { + for _, h := range hosts { + if g, err := glob.Compile(h); err == nil { + globs = append(globs, g) + } else { + log.Error("glob.Compile %s failed: %v", h, err) } - return os.Getenv("https_proxy") } - return setting.Proxy.ProxyURL + return globs } -// Match return true if url needs to be proxied -func Match(u string) bool { - if !setting.Proxy.Enabled { - return false - } - - // enforce do once - Proxy() - - for _, v := range hostMatchers { - if v.Match(u) { - return true - } - } - return false -} - -// Proxy returns the system proxy -func Proxy() func(req *http.Request) (*url.URL, error) { - if !setting.Proxy.Enabled { - return func(req *http.Request) (*url.URL, error) { - return nil, nil - } - } - if setting.Proxy.ProxyURL == "" { - return http.ProxyFromEnvironment - } - - once.Do(func() { - for _, h := range setting.Proxy.ProxyHosts { - if g, err := glob.Compile(h); err == nil { - hostMatchers = append(hostMatchers, g) - } else { - log.Error("glob.Compile %s failed: %v", h, err) - } - } - }) - +// hostsProxy sends requests for hosts matching globs through proxyURL, others follow the environment +func hostsProxy(globs []glob.Glob, proxyURL *url.URL) func(req *http.Request) (*url.URL, error) { return func(req *http.Request) (*url.URL, error) { - for _, v := range hostMatchers { - if v.Match(req.URL.Host) { - return http.ProxyURL(setting.Proxy.ProxyURLFixed)(req) + for _, g := range globs { + if g.Match(req.URL.Host) { + return proxyURL, nil } } return http.ProxyFromEnvironment(req) } } -// EnvWithProxy returns os.Environ(), with a https_proxy env, if the given url -// needs to be proxied. -func EnvWithProxy(u *url.URL) []string { - envs := os.Environ() - if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { - if Match(u.Host) { - envs = append(envs, "https_proxy="+GetProxyURL()) - } +// Proxy returns the system proxy +func Proxy() func(req *http.Request) (*url.URL, error) { + if !setting.Proxy.Enabled { + return nil } - - return envs + if setting.Proxy.ProxyURL == "" { + return http.ProxyFromEnvironment + } + return hostsProxy(globalProxyHosts(), setting.Proxy.ProxyURLFixed) +} + +// WebHookProxy returns the webhook proxy, falling back to the system proxy if no webhook proxy is set +func WebHookProxy() func(req *http.Request) (*url.URL, error) { + if setting.Webhook.ProxyURL == "" { + return Proxy() + } + return hostsProxy(webhookProxyHosts(), setting.Webhook.ProxyURLFixed) } diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 5a6079b6e2d..1833c259492 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -3,17 +3,20 @@ package setting +import "strings" + // Migrations settings var Migrations = struct { - MaxAttempts int - RetryBackoff int - AllowedDomains string - BlockedDomains string - AllowLocalNetworks bool - SkipTLSVerify bool + MaxAttempts int + RetryBackoff int + EgressMode string + AllowedHostList string + BlockedHostList string + SkipTLSVerify bool }{ MaxAttempts: 3, RetryBackoff: 3, + EgressMode: "lax", } func loadMigrationsFrom(rootCfg ConfigProvider) { @@ -21,8 +24,31 @@ func loadMigrationsFrom(rootCfg ConfigProvider) { Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) - Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").MustString("") - Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").MustString("") - Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) + egressModeSet := sec.HasKey("EGRESS_MODE") + Migrations.EgressMode = normalizePolicyMode(sec.Key("EGRESS_MODE").String()) + + deprecatedSetting(rootCfg, "migrations", "ALLOWED_DOMAINS", "migrations", "ALLOWED_HOST_LIST", "v28.0.0") + deprecatedSetting(rootCfg, "migrations", "BLOCKED_DOMAINS", "migrations", "BLOCKED_HOST_LIST", "v28.0.0") + deprecatedSetting(rootCfg, "migrations", "ALLOW_LOCALNETWORKS", "migrations", "ALLOWED_HOST_LIST", "v28.0.0") + Migrations.AllowedHostList = ConfigSectionKeyString(sec, "ALLOWED_HOST_LIST") + if Migrations.AllowedHostList == "" { + var hosts []string + for host := range strings.SplitSeq(ConfigSectionKeyString(sec, "ALLOWED_DOMAINS"), ",") { + if host = strings.TrimSpace(host); host != "" { + hosts = append(hosts, host+":*") + } + } + if len(hosts) > 0 && !egressModeSet { + Migrations.EgressMode = "strict" // ALLOWED_DOMAINS allowed only its hosts, on any port + } + if ConfigSectionKeyBool(sec, "ALLOW_LOCALNETWORKS") { + hosts = append(hosts, "private:*", "loopback:*") + } + Migrations.AllowedHostList = strings.Join(hosts, ",") + } + Migrations.BlockedHostList = ConfigSectionKeyString(sec, "BLOCKED_HOST_LIST", ConfigSectionKeyString(sec, "BLOCKED_DOMAINS")) + checkHostList("[migrations] ALLOWED_HOST_LIST", Migrations.AllowedHostList, false) + checkHostList("[migrations] BLOCKED_HOST_LIST", Migrations.BlockedHostList, true) + Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false) } diff --git a/modules/setting/migrations_test.go b/modules/setting/migrations_test.go new file mode 100644 index 00000000000..341ac3a1ec3 --- /dev/null +++ b/modules/setting/migrations_test.go @@ -0,0 +1,33 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "gitea.dev/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadMigrationsFrom(t *testing.T) { + defer test.MockVariableValue(&Migrations)() + for ini, want := range map[string][3]string{ + `ALLOWED_DOMAINS = github.com +BLOCKED_DOMAINS = gitlab.com +ALLOW_LOCALNETWORKS = true`: {"github.com:*,private:*,loopback:*", "gitlab.com", "strict"}, + `ALLOW_LOCALNETWORKS = true`: {"private:*,loopback:*", "", "lax"}, + `ALLOWED_HOST_LIST = 10.0.0.0/8 +ALLOWED_DOMAINS = github.com +ALLOW_LOCALNETWORKS = true +BLOCKED_HOST_LIST = evil.com +BLOCKED_DOMAINS = gitlab.com`: {"10.0.0.0/8", "evil.com", "lax"}, + } { + cfg, err := NewConfigProviderFromData("[migrations]\n" + ini) + require.NoError(t, err) + loadMigrationsFrom(cfg) + assert.Equal(t, want, [3]string{Migrations.AllowedHostList, Migrations.BlockedHostList, Migrations.EgressMode}, ini) + } +} diff --git a/modules/setting/security.go b/modules/setting/security.go index e8f5e1f2abe..1c514af5a5c 100644 --- a/modules/setting/security.go +++ b/modules/setting/security.go @@ -9,6 +9,7 @@ import ( "strings" "gitea.dev/modules/auth/password/hash" + "gitea.dev/modules/egress/policy" "gitea.dev/modules/generate" "gitea.dev/modules/log" ) @@ -20,11 +21,37 @@ var Security = struct { XContentTypeOptions string ContentSecurityPolicyGeneral string // it only supports empty (default policy) or "unset", maybe it can support more in the future + EgressMode string AllowedHostList string }{ XFrameOptions: "SAMEORIGIN", XContentTypeOptions: "nosniff", - AllowedHostList: "external", + EgressMode: "lax", +} + +// normalizePolicyMode validates a lax/strict egress policy EGRESS_MODE value, empty defaults to lax +func normalizePolicyMode(mode string) string { + mode = strings.ToLower(strings.TrimSpace(mode)) + switch mode { + case "": + return "lax" + case "lax", "strict": + return mode + default: + log.Fatal("Invalid egress policy EGRESS_MODE %q, use lax or strict", mode) + return "" + } +} + +// checkHostList reports the entries an egress host list drops, a dropped block entry would allow a blocked host so it stops startup +func checkHostList(key, hostList string, isBlockList bool) { + rejected := policy.NewAllowList(hostList, policy.Lax).Rejected() + for _, reason := range rejected { + LogStartupProblem(1, log.ERROR, "%s ignores an invalid entry: %s", key, reason) + } + if isBlockList && len(rejected) > 0 { + log.Fatal("%s has invalid entries, fix them so no blocked host is allowed", key) + } } var ( @@ -160,6 +187,12 @@ func loadSecurityFrom(rootCfg ConfigProvider) { if err := sec.MapTo(&Security); err != nil { log.Fatal("Failed to map security settings: %v", err) } + egressModeSet := sec.HasKey("EGRESS_MODE") + Security.EgressMode = normalizePolicyMode(sec.Key("EGRESS_MODE").String()) + checkHostList("[security] ALLOWED_HOST_LIST", Security.AllowedHostList, false) + if Security.AllowedHostList != "" && !egressModeSet { + LogStartupProblem(1, log.WARN, "[security] ALLOWED_HOST_LIST only restricts private hosts in the default lax mode, set EGRESS_MODE = strict to allow only the listed hosts, or lax to keep this") + } twoFactorAuth := sec.Key("TWO_FACTOR_AUTH").String() switch twoFactorAuth { diff --git a/modules/setting/security_test.go b/modules/setting/security_test.go index d830a517945..447edbc21e7 100644 --- a/modules/setting/security_test.go +++ b/modules/setting/security_test.go @@ -12,11 +12,13 @@ import ( func TestLoadSecurityFrom(t *testing.T) { assert.Equal(t, "SAMEORIGIN", Security.XFrameOptions) assert.Equal(t, "nosniff", Security.XContentTypeOptions) - assert.Equal(t, "external", Security.AllowedHostList) + assert.Equal(t, "lax", Security.EgressMode) + assert.Equal(t, "", Security.AllowedHostList) cfg, err := NewConfigProviderFromData(`[security] X_FRAME_OPTIONS = DENY X_CONTENT_TYPE_OPTIONS = unset +EGRESS_MODE = Strict ALLOWED_HOST_LIST = foo CONTENT_SECURITY_POLICY_GENERAL = "script-src *; foo" `) @@ -24,6 +26,7 @@ CONTENT_SECURITY_POLICY_GENERAL = "script-src *; foo" loadSecurityFrom(cfg) assert.Equal(t, "DENY", Security.XFrameOptions) assert.Equal(t, "unset", Security.XContentTypeOptions) + assert.Equal(t, "strict", Security.EgressMode) assert.Equal(t, "foo", Security.AllowedHostList) assert.Equal(t, `"script-src *`, Security.ContentSecurityPolicyGeneral) // holy shit ini package bug } diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index 18df3084798..a192c5b49b9 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -37,6 +37,9 @@ func loadWebhookFrom(rootCfg ConfigProvider) { deprecatedSetting(rootCfg, "webhook", "ALLOWED_HOST_LIST", "security", "ALLOWED_HOST_LIST", "v28.0.0") Webhook.AllowedHostList = sec.Key("ALLOWED_HOST_LIST").MustString(Security.AllowedHostList) + if Webhook.AllowedHostList != Security.AllowedHostList { + checkHostList("[webhook] ALLOWED_HOST_LIST", Webhook.AllowedHostList, false) + } Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork", "packagist"} Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10) diff --git a/options/locale/locale_en-US.json b/options/locale/locale_en-US.json index 5f1de7335e9..c6e574774ba 100644 --- a/options/locale/locale_en-US.json +++ b/options/locale/locale_en-US.json @@ -1102,7 +1102,7 @@ "repo.migrate.github_token_desc": "You can put one or more tokens here, separated by commas, to make migrating faster by circumventing the GitHub API rate limit. WARNING: Abusing this feature may violate the service provider's policy and may lead to getting your account(s) blocked.", "repo.migrate.clone_local_path": "or a local server path", "repo.migrate.permission_denied": "You are not allowed to import local repositories.", - "repo.migrate.permission_denied_blocked": "You cannot import from disallowed hosts. Please ask the admin to check ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS settings.", + "repo.migrate.permission_denied_blocked": "You cannot import from disallowed hosts. Please ask the admin to check the ALLOWED_HOST_LIST/BLOCKED_HOST_LIST settings in the [migrations] section.", "repo.migrate.invalid_local_path": "The local path is invalid. It doesn't exist or is not a directory.", "repo.migrate.invalid_lfs_endpoint": "The LFS endpoint is not valid.", "repo.migrate.failed": "Migration failed: %v", diff --git a/routers/init.go b/routers/init.go index c4e415893b2..7770243e434 100644 --- a/routers/init.go +++ b/routers/init.go @@ -39,11 +39,11 @@ import ( "gitea.dev/services/automerge" "gitea.dev/services/cron" feed_service "gitea.dev/services/feed" + gitproxy_service "gitea.dev/services/gitproxy" indexer_service "gitea.dev/services/indexer" "gitea.dev/services/mailer" mailer_incoming "gitea.dev/services/mailer/incoming" markup_service "gitea.dev/services/markup" - repo_migrations "gitea.dev/services/migrations" mirror_service "gitea.dev/services/mirror" "gitea.dev/services/oauth2_provider" packages_spec "gitea.dev/services/packages/pkgspec" @@ -147,6 +147,7 @@ func InitWebInstalled(ctx context.Context) { mustInit(packages_spec.InitManager) // Booting long running goroutines. + mustInitCtx(ctx, gitproxy_service.Run) // must start before mirror/migration services spawn git mustInit(indexer_service.Init) mirror_service.InitSyncMirrors() @@ -154,7 +155,6 @@ func InitWebInstalled(ctx context.Context) { mustInit(pull_service.Init) mustInitCtx(ctx, automerge.Init) mustInit(task.Init) - mustInit(repo_migrations.Init) mustInit(websocket_service.Init) mustInitCtx(ctx, mailer_incoming.Init) diff --git a/routers/web/auth/oauth.go b/routers/web/auth/oauth.go index 0192aa1bd50..4e0a355d87e 100644 --- a/routers/web/auth/oauth.go +++ b/routers/web/auth/oauth.go @@ -19,11 +19,10 @@ import ( user_model "gitea.dev/models/user" auth_module "gitea.dev/modules/auth" "gitea.dev/modules/container" - "gitea.dev/modules/hostmatcher" + "gitea.dev/modules/egress" "gitea.dev/modules/httplib" "gitea.dev/modules/log" "gitea.dev/modules/optional" - "gitea.dev/modules/proxy" "gitea.dev/modules/session" "gitea.dev/modules/setting" "gitea.dev/services/audit" @@ -300,21 +299,10 @@ func showLinkingLogin(ctx *context.Context, authSourceID int64, gothUser goth.Us ctx.Redirect(setting.AppSubURL + "/user/link_account") } -// oauth2AvatarAllowList parses the host allow-list applied to avatar fetches from the global -// [security] ALLOWED_HOST_LIST, defaulting an empty setting to the built-in "external" set. An empty -// host-match list would otherwise disable the allow-list check entirely and permit any host, including -// loopback/private addresses (SSRF). -func oauth2AvatarAllowList() *hostmatcher.HostMatchList { - return hostmatcher.ParseHostMatchList("security.ALLOWED_HOST_LIST", setting.Security.AllowedHostList) -} - -// oauth2AvatarHTTPClient builds the SSRF-protected client for avatar fetches. It is constructed per call -// so a changed allowlist takes effect (avatar fetches are infrequent, so this is not a hot path). func oauth2AvatarHTTPClient() *http.Client { - allowList := oauth2AvatarAllowList() return &http.Client{ Timeout: 30 * time.Second, - Transport: hostmatcher.NewHTTPTransport("oauth2-avatar", allowList, nil, proxy.Proxy(), setting.Proxy.ProxyURLFixed, nil), + Transport: egress.NewSecurityPolicy("oauth2-avatar").NewHTTPTransport(), } } diff --git a/routers/web/auth/oauth_test.go b/routers/web/auth/oauth_test.go index 36e4445ad84..38ee20f41fe 100644 --- a/routers/web/auth/oauth_test.go +++ b/routers/web/auth/oauth_test.go @@ -12,9 +12,7 @@ import ( "gitea.dev/models/auth" "gitea.dev/models/unittest" user_model "gitea.dev/models/user" - "gitea.dev/modules/hostmatcher" - "gitea.dev/modules/setting" - "gitea.dev/modules/test" + "gitea.dev/modules/egress/policy" "gitea.dev/services/oauth2_provider" "github.com/golang-jwt/jwt/v5" @@ -98,24 +96,12 @@ func TestOAuth2AvatarClientBlocksLoopback(t *testing.T) { assert.False(t, hit.Load(), "avatar client must refuse to dial a loopback address") } -func TestOAuth2AvatarAllowListRestricts(t *testing.T) { - defer test.MockVariableValue(&setting.Security.AllowedHostList, "avatars.example.com")() - allowList := oauth2AvatarAllowList() - assert.True(t, allowList.MatchHostName("avatars.example.com"), "the configured host must be allowed") - assert.False(t, allowList.MatchHostName("8.8.8.8"), "an unrelated external host must be rejected") - - // the default `external` allow-list still permits external hosts - setting.Security.AllowedHostList = hostmatcher.MatchBuiltinExternal - assert.True(t, oauth2AvatarAllowList().MatchHostName("8.8.8.8"), "default allow-list permits external hosts") -} - func TestOAuth2AvatarClientBlocksCloudMetadata(t *testing.T) { - // external-only allow-list must reject link-local cloud metadata (169.254.169.254) at dial time resp, err := oauth2AvatarHTTPClient().Get("http://169.254.169.254/latest/meta-data/") if resp != nil { _ = resp.Body.Close() } require.Error(t, err) - assert.ErrorContains(t, err, "can only call allowed HTTP servers", + assert.ErrorIs(t, err, policy.ErrDenied, "avatar client must refuse a link-local cloud-metadata address") } diff --git a/services/gitproxy/server.go b/services/gitproxy/server.go new file mode 100644 index 00000000000..af52a14ee21 --- /dev/null +++ b/services/gitproxy/server.go @@ -0,0 +1,397 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitproxy + +import ( + "bufio" + "cmp" + "context" + "crypto/rand" + "crypto/subtle" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httputil" + "net/netip" + "net/url" + "os" + "strings" + "time" + + "gitea.dev/modules/egress" + "gitea.dev/modules/egress/policy" + "gitea.dev/modules/git/gitcmd" + "gitea.dev/modules/log" + "gitea.dev/modules/setting" + + "github.com/Azure/go-ntlmssp" + "golang.org/x/net/proxy" +) + +const ( + proxyURLEnv = "GITEA_GIT_PROXY" // tells this binary it runs as git's GIT_PROXY_COMMAND + directHeader = "X-Gitea-Direct" // asks for a CONNECT tunnel that skips the operator's proxy, as git:// remotes never used one +) + +// proxyDialer reaches the operator's proxies, which are configuration rather than user input +var proxyDialer = &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} + +// copyBuffers beats the reverse proxy's 32 KiB default as it flushes every write, the transport's ReadBufferSize lets a read span chunks +type copyBuffers struct{} + +func (copyBuffers) Get() []byte { return make([]byte, 256*1024) } +func (copyBuffers) Put([]byte) {} + +// server is a forward proxy for git's remotes that enforces an egress policy on its direct dials. +type server struct { + auth string // the Proxy-Authorization header git must send + policy *policy.Policy + dial func(ctx context.Context, network, addr string) (net.Conn, error) + reverseProxy *httputil.ReverseProxy + proxyTLS *tls.Config + proxyNTLM bool // CONNECT only, the transport can't pin the connection NTLM authenticates +} + +func newServer(p *policy.Policy, auth string, proxyTLS *tls.Config) *server { + s := &server{auth: auth, policy: p, dial: p.NewDialContext(), proxyTLS: cmp.Or(proxyTLS, &tls.Config{})} + transport := p.NewHTTPTransport() + transport.Proxy = s.upstreamProxy + transport.TLSClientConfig = s.proxyTLS.Clone() // the transport adds its ALPN protocols to the config it gets + transport.ReadBufferSize = 64 * 1024 + s.reverseProxy = &httputil.ReverseProxy{ + Rewrite: func(*httputil.ProxyRequest) {}, + Transport: transport, + FlushInterval: -1, + BufferPool: copyBuffers{}, + ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) { writeUpstreamError(w, err) }, + } + return s +} + +// Run routes git's network remotes through a proxy on a random loopback port until ctx is done. +func Run(ctx context.Context) error { + gitPolicy, err := egress.NewGitPolicy() + if err != nil { + return fmt.Errorf("git proxy: %w", err) + } + gitOption := func(key, env string) string { return cmp.Or(os.Getenv(env), setting.GitConfig.GetOption(key)) } + proxyTLS, err := proxyTLSConfig(gitOption("http.proxySSLCAInfo", "GIT_PROXY_SSL_CAINFO"), + gitOption("http.proxySSLCert", "GIT_PROXY_SSL_CERT"), gitOption("http.proxySSLKey", "GIT_PROXY_SSL_KEY")) + if err != nil { + return fmt.Errorf("git proxy: %w", err) + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return fmt.Errorf("git proxy: %w", err) + } + user := url.UserPassword("gitea", rand.Text()) + handler := newServer(gitPolicy, basicAuth(user), proxyTLS) + handler.proxyNTLM = strings.EqualFold(gitOption("http.proxyAuthMethod", "GIT_HTTP_PROXY_AUTHMETHOD"), "ntlm") + srv := &http.Server{Handler: handler, ReadHeaderTimeout: 10 * time.Second} + context.AfterFunc(ctx, func() { _ = srv.Close() }) + go func() { + if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) { + log.Error("git proxy: %v", err) + } + }() + gitcmd.SetExtraEnvs(gitEnvs((&url.URL{Scheme: "http", User: user, Host: ln.Addr().String()}).String())) + return nil +} + +// gitEnvs route git's http(s) remotes through proxyURL and its git:// remotes through MaybeTunnel, command scope config beats every config file and keeps the credentials out of process listings +func gitEnvs(proxyURL string) []string { + envs := []string{ + "GIT_CONFIG_PARAMETERS=" + strings.TrimSpace(os.Getenv("GIT_CONFIG_PARAMETERS")+" 'http.proxy="+proxyURL+"'"), + "GIT_HTTP_PROXY_AUTHMETHOD=basic", + "no_proxy=", "NO_PROXY=", // git honors no_proxy even for a configured proxy + } + if setting.GitConfig.GetOption("core.gitProxy") == "" { // the operator's own git:// proxy command stays in charge + envs = append(envs, "GIT_PROXY_COMMAND="+setting.AppPath, proxyURLEnv+"="+proxyURL) + } + return envs +} + +// MaybeTunnel serves as git's GIT_PROXY_COMMAND for git:// remotes when git runs this binary with host and port, it returns otherwise +func MaybeTunnel() { + proxyURL := os.Getenv(proxyURLEnv) + if proxyURL == "" || len(os.Args) != 3 { + return + } + if err := tunnel(proxyURL, net.JoinHostPort(os.Args[1], os.Args[2])); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) +} + +func tunnel(proxyURL, target string) error { + gitProxy, err := url.Parse(proxyURL) + if err != nil { + return err + } + conn, err := net.Dial("tcp", gitProxy.Host) + if err != nil { + return err + } + if conn, err = openTunnel(context.Background(), conn, gitProxy, target, http.Header{directHeader: {"1"}}, false); err != nil { + return err + } + go func() { + _, _ = io.Copy(conn, os.Stdin) + closeWrite(conn) + }() + _, err = io.Copy(os.Stdout, conn) + return err +} + +// proxyTLSConfig loads the files of git's http.proxySSL* options, like curl the CA file replaces the system roots +func proxyTLSConfig(caFile, certFile, keyFile string) (*tls.Config, error) { + cfg := &tls.Config{} + if caFile != "" { + pemData, err := os.ReadFile(caFile) + if err != nil { + return nil, err + } + cfg.RootCAs = x509.NewCertPool() + if !cfg.RootCAs.AppendCertsFromPEM(pemData) { + return nil, fmt.Errorf("no certificates in %s", caFile) + } + } + if certFile != "" { + cert, err := tls.LoadX509KeyPair(certFile, cmp.Or(keyFile, certFile)) + if err != nil { + return nil, err + } + cfg.Certificates = []tls.Certificate{cert} + } + return cfg, nil +} + +func basicAuth(user *url.Userinfo) string { + password, _ := user.Password() + return "Basic " + base64.StdEncoding.EncodeToString([]byte(user.Username()+":"+password)) +} + +func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if subtle.ConstantTimeCompare([]byte(r.Header.Get("Proxy-Authorization")), []byte(s.auth)) != 1 { + w.Header().Set("Proxy-Authenticate", `Basic realm="gitea egress proxy"`) + http.Error(w, "egress: proxy authentication required", http.StatusProxyAuthRequired) + return + } + switch { + case r.Method == http.MethodConnect: + s.handleConnect(w, r) + case r.URL.Scheme == "http" && r.URL.Host != "": + s.reverseProxy.ServeHTTP(w, r) + default: + http.Error(w, "egress: CONNECT or an absolute http URI required", http.StatusBadRequest) + } +} + +func (s *server) handleConnect(w http.ResponseWriter, r *http.Request) { + if host, _, err := net.SplitHostPort(r.URL.Host); err != nil || host == "" { + http.Error(w, "egress: invalid CONNECT target", http.StatusBadRequest) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + dial := s.dialUpstream + if r.Header.Get(directHeader) != "" { + dial = func(ctx context.Context, target string) (net.Conn, error) { return s.dial(ctx, "tcp", target) } + } + upstream, err := dial(ctx, r.URL.Host) + if err != nil { + writeUpstreamError(w, err) + return + } + client, buffered, err := http.NewResponseController(w).Hijack() + if err != nil { + _ = upstream.Close() + http.Error(w, "egress: "+err.Error(), http.StatusInternalServerError) + return + } + if _, err := client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil { + _ = client.Close() + _ = upstream.Close() + return + } + relay(withBuffered(client, buffered.Reader), upstream) +} + +// upstreamProxy selects the operator's proxy for req, local targets are dialed directly as they would name the proxy's own host +func (s *server) upstreamProxy(req *http.Request) (proxyURL *url.URL, err error) { + if !isLocalHost(req.URL.Hostname()) { + proxyURL, err = s.policy.Proxy(req) + } + return proxyURL, err +} + +func isLocalHost(host string) bool { + host = strings.TrimSuffix(strings.ToLower(host), ".") + if host == "localhost" || strings.HasSuffix(host, ".localhost") { + return true + } + ip, err := netip.ParseAddr(host) + return err == nil && (ip.Unmap().IsLoopback() || ip.IsUnspecified()) +} + +func (s *server) dialUpstream(ctx context.Context, target string) (net.Conn, error) { + proxyURL, err := s.upstreamProxy(&http.Request{URL: &url.URL{Scheme: "https", Host: strings.TrimSuffix(target, ":443")}}) + switch { + case err != nil: + return nil, err + case proxyURL == nil: + return s.dial(ctx, "tcp", target) + case proxyURL.Scheme == "socks5" || proxyURL.Scheme == "socks5h": + dialer, err := proxy.FromURL(proxyURL, proxyDialer) + if err != nil { + return nil, err + } + ctxDialer, ok := dialer.(proxy.ContextDialer) + if !ok { + return nil, errors.New("egress: socks dialer lacks context support") + } + return ctxDialer.DialContext(ctx, "tcp", target) + default: + return s.connectVia(ctx, proxyURL, target) + } +} + +func (s *server) connectVia(ctx context.Context, proxyURL *url.URL, target string) (net.Conn, error) { + conn, err := proxyDialer.DialContext(ctx, "tcp", policy.ProxyDialAddr(proxyURL)) + if err != nil { + return nil, err + } + if proxyURL.Scheme == "https" { + cfg := s.proxyTLS.Clone() + cfg.ServerName = proxyURL.Hostname() + conn = tls.Client(conn, cfg) + } + return openTunnel(ctx, conn, proxyURL, target, http.Header{}, s.proxyNTLM) +} + +// openTunnel opens a CONNECT tunnel to target over conn to the proxy at proxyURL, closing conn on failure +func openTunnel(ctx context.Context, conn net.Conn, proxyURL *url.URL, target string, header http.Header, ntlm bool) (_ net.Conn, err error) { + defer func() { + if err != nil { + _ = conn.Close() + } + }() + defer context.AfterFunc(ctx, func() { _ = conn.Close() })() + switch { + case ntlm: + negotiate, _ := ntlmssp.NewNegotiateMessage("", "") + header.Set("Proxy-Authorization", "NTLM "+base64.StdEncoding.EncodeToString(negotiate)) + case proxyURL.User != nil: + header.Set("Proxy-Authorization", basicAuth(proxyURL.User)) + } + req := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: header} + reader := bufio.NewReader(conn) + resp, err := roundTrip(conn, reader, req) + if err != nil { + return nil, err + } + if ntlm && resp.StatusCode == http.StatusProxyAuthRequired { + _ = resp.Body.Close() // drains it for the next request on the connection + authenticate, err := ntlmAuthenticate(resp.Header, proxyURL.User) + if err != nil { + return nil, err + } + header.Set("Proxy-Authorization", authenticate) + if resp, err = roundTrip(conn, reader, req); err != nil { + return nil, err + } + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode > 299 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return nil, fmt.Errorf("egress: proxy refused CONNECT: %s", strings.TrimSpace(resp.Status+" "+string(body))) + } + return withBuffered(conn, reader), nil +} + +func roundTrip(conn net.Conn, reader *bufio.Reader, req *http.Request) (*http.Response, error) { + if err := req.Write(conn); err != nil { + return nil, err + } + return http.ReadResponse(reader, req) +} + +func ntlmAuthenticate(header http.Header, user *url.Userinfo) (string, error) { + for _, value := range header.Values("Proxy-Authenticate") { + if encoded, ok := strings.CutPrefix(value, "NTLM "); ok { + challenge, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return "", err + } + password, _ := user.Password() + authenticate, err := ntlmssp.NewAuthenticateMessage(challenge, user.Username(), password, nil) + if err != nil { + return "", err + } + return "NTLM " + base64.StdEncoding.EncodeToString(authenticate), nil + } + } + return "", errors.New("egress: proxy sent no NTLM challenge") +} + +func writeUpstreamError(w http.ResponseWriter, err error) { + var netErr net.Error + switch { + case errors.Is(err, policy.ErrDenied): + http.Error(w, "egress: target denied by policy", http.StatusForbidden) + case errors.As(err, &netErr) && netErr.Timeout(): + http.Error(w, "egress: upstream timeout: "+err.Error(), http.StatusGatewayTimeout) + default: + http.Error(w, "egress: upstream failed: "+err.Error(), http.StatusBadGateway) + } +} + +// bufferedConn reads the bytes a handshake left buffered before the rest of the connection +type bufferedConn struct { + net.Conn + r io.Reader +} + +func (c *bufferedConn) Read(p []byte) (int, error) { return c.r.Read(p) } + +func (c *bufferedConn) CloseWrite() error { + closeWrite(c.Conn) + return nil +} + +func withBuffered(conn net.Conn, r *bufio.Reader) net.Conn { + if r.Buffered() == 0 { + return conn // a bare socket lets io.Copy splice + } + return &bufferedConn{Conn: conn, r: r} +} + +// relay copies both ways and passes each end of stream on, the git:// protocol needs the half-close +func relay(client, upstream net.Conn) { + done := make(chan struct{}) + go func() { + _, _ = io.Copy(upstream, client) + closeWrite(upstream) + close(done) + }() + _, _ = io.Copy(client, upstream) + closeWrite(client) + <-done + _ = client.Close() + _ = upstream.Close() +} + +func closeWrite(conn net.Conn) { + if cw, ok := conn.(interface{ CloseWrite() error }); ok { + _ = cw.CloseWrite() + } else { + _ = conn.Close() + } +} diff --git a/services/gitproxy/server_test.go b/services/gitproxy/server_test.go new file mode 100644 index 00000000000..1c367147fe5 --- /dev/null +++ b/services/gitproxy/server_test.go @@ -0,0 +1,444 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitproxy + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/binary" + "encoding/pem" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "testing" + + "gitea.dev/modules/egress/policy" + "gitea.dev/modules/git/gitcmd" + "gitea.dev/modules/setting" + "gitea.dev/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testAuth = "Basic dGVzdDp0ZXN0" + +var ( + allowLoopback = policy.NewPolicy("test", policy.Lax, policy.WithAllow("loopback", "")) + blockLoopback = policy.NewPolicy("test", policy.Lax, policy.WithBlock("loopback", "")) +) + +func listen(t *testing.T) net.Listener { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + return ln +} + +func serveConns(t *testing.T, handle func(net.Conn)) string { + ln := listen(t) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + handle(conn) + }() + } + }() + return ln.Addr().String() +} + +func startEcho(t *testing.T) string { + return serveConns(t, func(conn net.Conn) { _, _ = io.Copy(conn, conn) }) +} + +func startProxy(t *testing.T, srv *server) string { + proxySrv := httptest.NewServer(srv) + t.Cleanup(proxySrv.Close) + return proxySrv.Listener.Addr().String() +} + +func viaProxy(u *url.URL) *policy.Policy { + return policy.NewPolicy("test", policy.Lax, policy.WithAllow("loopback", ""), policy.WithProxy(http.ProxyURL(u))) +} + +func serve(srv *server, method, target, auth string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, target, nil) + req.Header.Set("Proxy-Authorization", auth) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + return rec +} + +func connect(t *testing.T, proxyAddr, target, auth string) (net.Conn, *bufio.Reader, int) { + conn, err := net.Dial("tcp", proxyAddr) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + _, err = fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\nProxy-Authorization: %s\r\n\r\n", target, target, auth) + require.NoError(t, err) + br := bufio.NewReader(conn) + resp, err := http.ReadResponse(br, &http.Request{Method: http.MethodConnect}) + require.NoError(t, err) + defer resp.Body.Close() + return conn, br, resp.StatusCode +} + +func assertEcho(t *testing.T, conn net.Conn, br *bufio.Reader) { + _, err := conn.Write([]byte("ping\n")) + require.NoError(t, err) + reply, err := br.ReadString('\n') + require.NoError(t, err) + assert.Equal(t, "ping\n", reply) +} + +func startConnectOperator(t *testing.T, useTLS bool, reply string, seen chan<- *http.Request) *httptest.Server { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen <- r + conn, buf, err := http.NewResponseController(w).Hijack() + if err != nil { + return + } + defer conn.Close() + _, _ = buf.WriteString(reply) + _ = buf.Flush() + _, _ = io.Copy(conn, buf) + }) + operator := httptest.NewUnstartedServer(handler) + if useTLS { + operator.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert} + operator.StartTLS() + } else { + operator.Start() + } + t.Cleanup(operator.Close) + return operator +} + +func startSOCKS5(t *testing.T) string { + return serveConns(t, func(conn net.Conn) { + br := bufio.NewReader(conn) + read := func(n int) []byte { + buf := make([]byte, n) + _, _ = io.ReadFull(br, buf) + return buf + } + _, _ = conn.Write([]byte{5, 2}) + _ = read(int(read(2)[1])) + gotUser := string(read(int(read(2)[1]))) + if gotPass := string(read(int(read(1)[0]))); gotUser != "user" || gotPass != "secret" { + _, _ = conn.Write([]byte{1, 1}) + return + } + _, _ = conn.Write([]byte{1, 0}) + _ = read(int(read(5)[4]) + 2) + _, _ = conn.Write([]byte{5, 0, 0, 1, 0, 0, 0, 0, 0, 0}) + _, _ = io.Copy(conn, br) + }) +} + +func TestUpstreamProxy(t *testing.T) { + t.Parallel() + var seen []string + errStop := errors.New("stop") + s := &server{policy: policy.NewPolicy("test", policy.Lax, policy.WithProxy(func(r *http.Request) (*url.URL, error) { + seen = append(seen, r.URL.Host) + return nil, errStop + }))} + for _, target := range []string{"github.com:443", "github.com:8443", "[2001:db8::1]:443"} { + _, err := s.dialUpstream(t.Context(), target) + assert.ErrorIs(t, err, errStop) + } + assert.Equal(t, []string{"github.com", "github.com:8443", "[2001:db8::1]"}, seen) + + for _, host := range []string{"localhost", "git.localhost.", "127.0.0.2", "::1", "::ffff:127.0.0.1", "0.0.0.0", "::"} { + u, err := s.upstreamProxy(&http.Request{URL: &url.URL{Host: net.JoinHostPort(host, "80")}}) + assert.NoError(t, err, host) + assert.Nil(t, u, host) + } +} + +func TestWriteUpstreamError(t *testing.T) { + t.Parallel() + for err, want := range map[error]int{ + fmt.Errorf("dial: %w", policy.ErrDenied): http.StatusForbidden, + context.DeadlineExceeded: http.StatusGatewayTimeout, + &net.OpError{Op: "dial", Net: "tcp", Err: os.ErrDeadlineExceeded}: http.StatusGatewayTimeout, + errors.New("refused"): http.StatusBadGateway, + } { + rec := httptest.NewRecorder() + writeUpstreamError(rec, err) + assert.Equal(t, want, rec.Code, err.Error()) + } +} + +func TestRelay(t *testing.T) { + t.Parallel() + client, clientEnd := net.Pipe() + upstreamEnd, upstream := net.Pipe() + done := make(chan struct{}) + go func() { + relay(clientEnd, upstreamEnd) + close(done) + }() + require.NoError(t, upstream.Close()) + _, err := client.Read(make([]byte, 1)) + assert.ErrorIs(t, err, io.EOF) + <-done +} + +func TestProxyCONNECT(t *testing.T) { + t.Parallel() + echo := startEcho(t) + srv := newServer(allowLoopback, testAuth, nil) + + assert.Equal(t, http.StatusProxyAuthRequired, serve(srv, http.MethodConnect, echo, "").Code) + assert.Equal(t, http.StatusForbidden, serve(newServer(blockLoopback, "", nil), http.MethodConnect, echo, "").Code) + assert.Equal(t, http.StatusBadRequest, serve(srv, http.MethodConnect, "127.0.0.1", testAuth).Code) + + conn, br, status := connect(t, startProxy(t, srv), echo, testAuth) + require.Equal(t, http.StatusOK, status) + assertEcho(t, conn, br) +} + +func TestProxyCONNECTOperator(t *testing.T) { + t.Parallel() + const target = "git.example.com:443" + + t.Run("HTTP", func(t *testing.T) { + t.Parallel() + seen := make(chan *http.Request, 1) + operator := startConnectOperator(t, false, "HTTP/1.1 204 No Content\r\n\r\nEARLY", seen) + opURL, err := url.Parse(operator.URL) + require.NoError(t, err) + opURL.User = url.UserPassword("user", "secret") + + conn, br, status := connect(t, startProxy(t, newServer(viaProxy(opURL), "", nil)), target, "") + require.Equal(t, http.StatusOK, status) + early := make([]byte, 5) + _, err = io.ReadFull(br, early) + require.NoError(t, err) + assert.Equal(t, "EARLY", string(early)) + assertEcho(t, conn, br) + req := <-seen + assert.Equal(t, target, req.Host) + assert.Equal(t, basicAuth(opURL.User), req.Header.Get("Proxy-Authorization")) + }) + + t.Run("HTTPS", func(t *testing.T) { + t.Parallel() + operator := startConnectOperator(t, true, "HTTP/1.1 200 OK\r\n\r\n", make(chan *http.Request, 1)) + opURL, err := url.Parse(operator.URL) + require.NoError(t, err) + key, err := x509.MarshalPKCS8PrivateKey(operator.TLS.Certificates[0].PrivateKey) + require.NoError(t, err) + pemFile := filepath.Join(t.TempDir(), "proxy.pem") + pemData := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: operator.Certificate().Raw}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key})...) + require.NoError(t, os.WriteFile(pemFile, pemData, 0o600)) + proxyTLS, err := proxyTLSConfig(pemFile, pemFile, "") + require.NoError(t, err) + + conn, br, status := connect(t, startProxy(t, newServer(viaProxy(opURL), "", proxyTLS)), target, "") + require.Equal(t, http.StatusOK, status) + assertEcho(t, conn, br) + }) + + t.Run("SOCKS5", func(t *testing.T) { + t.Parallel() + opURL := &url.URL{Scheme: "socks5", User: url.UserPassword("user", "secret"), Host: startSOCKS5(t)} + conn, br, status := connect(t, startProxy(t, newServer(viaProxy(opURL), "", nil)), target, "") + require.Equal(t, http.StatusOK, status) + assertEcho(t, conn, br) + }) + + t.Run("NTLM", func(t *testing.T) { + t.Parallel() + challenge := make([]byte, 48) + copy(challenge, "NTLMSSP\x00") + challenge[8] = 2 + binary.LittleEndian.PutUint32(challenge[20:], 0x201) + auths := make(chan string, 2) + addr := serveConns(t, func(conn net.Conn) { + br := bufio.NewReader(conn) + for _, reply := range []string{"HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: NTLM " + base64.StdEncoding.EncodeToString(challenge) + "\r\nContent-Length: 4\r\n\r\ndeny", "HTTP/1.1 200 OK\r\n\r\n"} { + req, err := http.ReadRequest(br) + if err != nil { + return + } + auths <- req.Header.Get("Proxy-Authorization") + _, _ = io.WriteString(conn, reply) + } + }) + srv := newServer(viaProxy(&url.URL{Scheme: "http", User: url.UserPassword(`CORP\alice`, "secret"), Host: addr}), "", nil) + srv.proxyNTLM = true + + _, _, status := connect(t, startProxy(t, srv), target, "") + require.Equal(t, http.StatusOK, status) + assert.Regexp(t, "^NTLM TlRMTVNTUAAB", <-auths) + assert.Regexp(t, "^NTLM TlRMTVNTUAAD", <-auths) + }) + + t.Run("Refused", func(t *testing.T) { + t.Parallel() + operator := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "blocked by policy", http.StatusForbidden) + })) + t.Cleanup(operator.Close) + opURL, err := url.Parse(operator.URL) + require.NoError(t, err) + + rec := serve(newServer(viaProxy(opURL), "", nil), http.MethodConnect, target, "") + assert.Equal(t, http.StatusBadGateway, rec.Code) + assert.Contains(t, rec.Body.String(), "blocked by policy") + }) + + t.Run("ClientGone", func(t *testing.T) { + t.Parallel() + ln := listen(t) + operatorClosed := make(chan struct{}) + requestSeen := make(chan struct{}) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer close(operatorClosed) + defer conn.Close() + if _, err := http.ReadRequest(bufio.NewReader(conn)); err != nil { + return + } + close(requestSeen) + _, _ = io.Copy(io.Discard, conn) + }() + + conn, err := net.Dial("tcp", startProxy(t, newServer(viaProxy(&url.URL{Scheme: "http", Host: ln.Addr().String()}), "", nil))) + require.NoError(t, err) + _, err = fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", target, target) + require.NoError(t, err) + responded := make(chan struct{}) + go func() { + _, _ = conn.Read(make([]byte, 1)) + close(responded) + }() + select { + case <-requestSeen: + case <-responded: + t.Fatal("proxy answered before reaching the operator") + } + require.NoError(t, conn.Close()) + <-operatorClosed + }) +} + +func TestProxyHTTP(t *testing.T) { + t.Parallel() + + t.Run("Forward", func(t *testing.T) { + t.Parallel() + firstRead := make(chan struct{}) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("X-Drop-Me")) + assert.False(t, r.Close) + w.Header().Set("Connection", "X-Origin-Drop") + w.Header().Set("X-Origin-Drop", "dropped") + _, _ = w.Write([]byte("0008NAK\n")) + http.NewResponseController(w).Flush() + <-firstRead + _, _ = w.Write([]byte("0000")) + })) + t.Cleanup(origin.Close) + + req, err := http.NewRequest(http.MethodGet, origin.URL+"/git-upload-pack", nil) + require.NoError(t, err) + req.Header.Set("Connection", "close, X-Drop-Me") + req.Header.Set("X-Drop-Me", "dropped") + proxyURL := &url.URL{Scheme: "http", Host: startProxy(t, newServer(allowLoopback, "", nil))} + resp, err := (&http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}).Do(req) + require.NoError(t, err) + defer resp.Body.Close() + assert.Empty(t, resp.Header.Get("X-Origin-Drop")) + + buf := make([]byte, 8) + _, err = io.ReadFull(resp.Body, buf) + require.NoError(t, err) + assert.Equal(t, "0008NAK\n", string(buf)) + close(firstRead) + rest, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, "0000", string(rest)) + }) + + t.Run("ViaOperator", func(t *testing.T) { + t.Parallel() + operator := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(r.URL.String())) + })) + t.Cleanup(operator.Close) + opURL, err := url.Parse(operator.URL) + require.NoError(t, err) + + rec := serve(newServer(viaProxy(opURL), "", nil), http.MethodGet, "http://git.example.com/repo.git", "") + assert.Equal(t, "http://git.example.com/repo.git", rec.Body.String()) + }) + + t.Run("Denied", func(t *testing.T) { + t.Parallel() + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("the blocked origin must not be reached") + })) + t.Cleanup(origin.Close) + assert.Equal(t, http.StatusForbidden, serve(newServer(blockLoopback, "", nil), http.MethodGet, origin.URL, "").Code) + }) + + t.Run("OriginForm", func(t *testing.T) { + t.Parallel() + assert.Equal(t, http.StatusBadRequest, serve(newServer(allowLoopback, "", nil), http.MethodGet, "/", "").Code) + }) +} + +func TestMain(m *testing.M) { + MaybeTunnel() + m.Run() +} + +func TestRun(t *testing.T) { + base := t.TempDir() + exe, err := os.Executable() + require.NoError(t, err) + repos, err := filepath.Abs("../../modules/git/tests/repos") + require.NoError(t, err) + defer test.MockVariableValue(&setting.Git.HomePath, base)() + defer test.MockVariableValue(&setting.AppPath, exe)() + defer test.MockVariableValue(&setting.Migrations.AllowedHostList, "127.0.0.1/32")() + t.Cleanup(func() { gitcmd.SetExtraEnvs(nil) }) + require.NoError(t, Run(t.Context())) + + stdout, _, runErr := gitcmd.NewCommand("config", "--get", "http.proxy").RunStdString(t.Context()) + require.NoError(t, runErr) + assert.Contains(t, stdout, "http://gitea:") + + _, port, err := net.SplitHostPort(serveConns(t, func(conn net.Conn) { + daemon := exec.Command("git", "daemon", "--inetd", "--export-all", "--base-path="+repos) + daemon.Stdin, daemon.Stdout = conn, conn + _ = daemon.Run() + })) + require.NoError(t, err) + require.NoError(t, gitcmd.NewCommand("clone", "-q", "--bare").AddDynamicArguments("git://127.0.0.1:"+port+"/repo1_bare", filepath.Join(base, "allowed")).Run(t.Context())) + _, stderr, runErr := gitcmd.NewCommand("clone", "-q", "--bare").AddDynamicArguments("git://127.0.0.2:"+port+"/repo1_bare", filepath.Join(base, "denied")).RunStdString(t.Context()) + require.Error(t, runErr) + assert.Contains(t, stderr, "target denied by policy") +} diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index 97c6cfe7aa5..ae223281f36 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -15,7 +15,6 @@ import ( "gitea.dev/modules/log" base "gitea.dev/modules/migration" - "gitea.dev/modules/proxy" "gitea.dev/modules/structs" ) @@ -79,6 +78,7 @@ type CodebaseDownloader struct { // NewCodebaseDownloader creates a new downloader func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader { baseURL, _ := url.Parse("https://api3.codebasehq.com") + transport := NewMigrationHTTPTransport() downloader := &CodebaseDownloader{ baseURL: baseURL, @@ -86,14 +86,13 @@ func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repo project: project, repoName: repoName, client: &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - if len(username) > 0 && len(password) > 0 { - req.SetBasicAuth(username, password) - } - return proxy.Proxy()(req) - }, - }, + Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + if username != "" && password != "" { + req = req.Clone(req.Context()) + req.SetBasicAuth(username, password) + } + return transport.RoundTrip(req) + }), }, userMap: make(map[int64]*codebaseUser), commitMap: make(map[string]string), diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index a76b0293969..9463b8c2d9a 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -948,7 +948,7 @@ func (g *GiteaLocalUploader) CreateReviews(ctx context.Context, reviews ...*base // Rollback when migrating failed, this will rollback all the changes. func (g *GiteaLocalUploader) Rollback() error { - if g.repo != nil && g.repo.ID > 0 { + if g.repo != nil && g.repo.ID > 0 && g.gitRepo != nil { g.gitRepo.Close() // do not delete the repository, otherwise the end users won't be able to see the last error message diff --git a/services/migrations/github.go b/services/migrations/github.go index 4ded46a92d8..3da289066a7 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -17,7 +17,6 @@ import ( "gitea.dev/modules/git" "gitea.dev/modules/log" base "gitea.dev/modules/migration" - "gitea.dev/modules/proxy" "gitea.dev/modules/structs" "github.com/google/go-github/v92/github" @@ -107,13 +106,8 @@ func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token } } } else { - transport := NewMigrationHTTPTransport() - transport.Proxy = func(req *http.Request) (*url.URL, error) { - req.SetBasicAuth(userName, password) - return proxy.Proxy()(req) - } client := &http.Client{ - Transport: transport, + Transport: &github.BasicAuthTransport{Transport: NewMigrationHTTPTransport(), Username: userName, Password: password}, } if err := downloader.addClient(client, baseURL); err != nil { return nil, err diff --git a/services/migrations/gogs.go b/services/migrations/gogs.go index 7a41851e145..f4b3b722688 100644 --- a/services/migrations/gogs.go +++ b/services/migrations/gogs.go @@ -107,11 +107,12 @@ func (g *GogsDownloader) client(ctx context.Context) *gogs.Client { gogsClient := gogs.NewClient(g.baseURL, g.token) gogsClient.SetHTTPClient(&http.Client{ Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + req = req.Clone(ctx) if g.password != "" { // Gogs client lacks the support for basic auth, this is the only way to set it req.SetBasicAuth(g.userName, g.password) } - return httpTransport.RoundTrip(req.WithContext(ctx)) + return httpTransport.RoundTrip(req) }), }) return gogsClient diff --git a/services/migrations/http_client.go b/services/migrations/http_client.go index 6fe7440c55b..38e9da76e46 100644 --- a/services/migrations/http_client.go +++ b/services/migrations/http_client.go @@ -6,37 +6,23 @@ package migrations import ( "crypto/tls" "net/http" + "sync" - "gitea.dev/modules/hostmatcher" - "gitea.dev/modules/proxy" + "gitea.dev/modules/egress" "gitea.dev/modules/setting" - "gitea.dev/modules/util" ) -// migrationHTTPClient is the shared migration client. Callers that would otherwise build a client per -// request use it (via getMigrationHTTPClient) so a single connection pool is reused across downloads — -// e.g. many release assets from the same host — instead of a fresh pool and TLS handshake each time. It -// is built lazily on first use and reset by Init whenever the allow/block lists change; OnceValue keeps -// concurrent callers sharing a single client instead of racing to create their own. -var migrationHTTPClient = util.OnceValue[*http.Client]{Func: newMigrationHTTPClient} +// getMigrationHTTPClient returns the shared migration client, so downloads from one host reuse its connections +var getMigrationHTTPClient = sync.OnceValue(newMigrationHTTPClient) // newMigrationHTTPClient returns a HTTP client for migration func newMigrationHTTPClient() *http.Client { - return &http.Client{ - Transport: NewMigrationHTTPTransport(), - } + return &http.Client{Transport: NewMigrationHTTPTransport()} } -// getMigrationHTTPClient returns the shared migration client, building it on first use so no request -// escapes the SSRF-validated transport even before Init has run. -func getMigrationHTTPClient() *http.Client { - return migrationHTTPClient.Value() -} - -// NewMigrationHTTPTransport returns a HTTP transport for migration. The target is validated against the -// allow/block lists on both the direct-dial and proxy paths, so a configured proxy cannot be used to -// reach an otherwise-forbidden target (SSRF). +// NewMigrationHTTPTransport returns a HTTP transport for migration, enforcing the migration policy on its direct dials. func NewMigrationHTTPTransport() *http.Transport { - return hostmatcher.NewHTTPTransport("migration", allowList, blockList, proxy.Proxy(), setting.Proxy.ProxyURLFixed, - &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}) + t := egress.NewMigrationPolicy().NewHTTPTransport() + t.TLSClientConfig = &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify} + return t } diff --git a/services/migrations/main_test.go b/services/migrations/main_test.go index d7230165a63..d4f3ca4f375 100644 --- a/services/migrations/main_test.go +++ b/services/migrations/main_test.go @@ -10,11 +10,13 @@ import ( "gitea.dev/models/unittest" base "gitea.dev/modules/migration" + "gitea.dev/modules/setting" "github.com/stretchr/testify/assert" ) func TestMain(m *testing.M) { + setting.Migrations.AllowedHostList = "private:*, loopback:*" unittest.MainTest(m) } diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 7a228465128..c85f103bc7a 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "net" "net/url" "path/filepath" "strings" @@ -17,8 +16,8 @@ import ( system_model "gitea.dev/models/system" user_model "gitea.dev/models/user" "gitea.dev/modules/container" + "gitea.dev/modules/egress" "gitea.dev/modules/git" - "gitea.dev/modules/hostmatcher" "gitea.dev/modules/log" base "gitea.dev/modules/migration" "gitea.dev/modules/setting" @@ -28,12 +27,7 @@ import ( // MigrateOptions is equal to base.MigrateOptions type MigrateOptions = base.MigrateOptions -var ( - factories []base.DownloaderFactory - - allowList *hostmatcher.HostMatchList - blockList *hostmatcher.HostMatchList -) +var factories []base.DownloaderFactory // RegisterDownloaderFactory registers a downloader factory func RegisterDownloaderFactory(factory base.DownloaderFactory) { @@ -76,33 +70,9 @@ func IsMigrateURLAllowed(remoteURL string, doer *user_model.User) error { return &git.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true} } - hostName, _, errIgnored := net.SplitHostPort(u.Host) - if errIgnored != nil { - hostName = u.Host // u.Host can be "host" or "host:port" + if err := egress.NewMigrationPolicy().CheckHostIPs(u); err != nil { + return &git.ErrInvalidCloneAddr{Host: u.Hostname(), IsPermissionDenied: true} } - - // some users only use proxy, there is no DNS resolver. it's safe to ignore the LookupIP error - addrList, _ := net.LookupIP(hostName) - return checkByAllowBlockList(hostName, addrList) -} - -func checkByAllowBlockList(hostName string, addrList []net.IP) error { - ipAllowed := len(addrList) > 0 - var ipBlocked bool - for _, addr := range addrList { - ipAllowed = ipAllowed && allowList.MatchIPAddr(addr) - ipBlocked = ipBlocked || blockList.MatchIPAddr(addr) - } - if blockList.MatchHostName(hostName) || ipBlocked { - return &git.ErrInvalidCloneAddr{Host: hostName, IsPermissionDenied: true} - } - // if we have an allow-list, check the allow-list before return to get the more accurate error - if !allowList.IsEmpty() { - if !allowList.MatchHostName(hostName) && !ipAllowed { - return &git.ErrInvalidCloneAddr{Host: hostName, IsPermissionDenied: true} - } - } - // otherwise, we always follow the blocked list return nil } @@ -509,29 +479,3 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba return uploader.Finish(ctx) } - -// Init migrations service -func Init() error { - // TODO: maybe we can deprecate these legacy ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS, use ALLOWED_HOST_LIST/BLOCKED_HOST_LIST instead - - blockList = hostmatcher.ParseSimpleMatchList("migrations.BLOCKED_DOMAINS", setting.Migrations.BlockedDomains) - - allowList = hostmatcher.ParseSimpleMatchList("migrations.ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS", setting.Migrations.AllowedDomains) - if allowList.IsEmpty() { - // the default policy is that migration module can access external hosts - allowList.AppendBuiltin(hostmatcher.MatchBuiltinExternal) - } - if setting.Migrations.AllowLocalNetworks { - allowList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate) - allowList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback) - } else { - blockList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate) - blockList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback) - } - - // reset the shared client so it is rebuilt from the freshly parsed lists on next use; download paths - // then reuse one connection pool instead of creating a client (and pool) per request - migrationHTTPClient.Reset() - - return nil -} diff --git a/services/migrations/migrate_test.go b/services/migrations/migrate_test.go index 8d421389688..4e3e86477b6 100644 --- a/services/migrations/migrate_test.go +++ b/services/migrations/migrate_test.go @@ -6,15 +6,14 @@ package migrations import ( "errors" "fmt" - "net" "net/http" "path/filepath" "testing" - "gitea.dev/models/unittest" user_model "gitea.dev/models/user" "gitea.dev/modules/git/gitcmd" "gitea.dev/modules/setting" + "gitea.dev/modules/test" "gitea.dev/modules/util" "github.com/google/go-github/v92/github" @@ -43,102 +42,29 @@ func TestIsAuthenticationError(t *testing.T) { } func TestMigrateWhiteBlocklist(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) + adminUser := &user_model.User{IsAdmin: true} + nonAdminUser := &user_model.User{} - adminUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user1"}) - nonAdminUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user2"}) - - setting.Migrations.AllowedDomains = "github.com" - setting.Migrations.AllowLocalNetworks = false - assert.NoError(t, Init()) - - err := IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser) - assert.Error(t, err) - - err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser) - assert.NoError(t, err) - - err = IsMigrateURLAllowed("https://gITHUb.com/go-gitea/gitea.git", nonAdminUser) - assert.NoError(t, err) - - setting.Migrations.AllowedDomains = "" - setting.Migrations.BlockedDomains = "github.com" - assert.NoError(t, Init()) - - err = IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser) - assert.NoError(t, err) - - err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser) - assert.Error(t, err) - - err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser) - assert.Error(t, err) - - setting.Migrations.AllowLocalNetworks = true - assert.NoError(t, Init()) - err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser) - assert.NoError(t, err) + defer test.MockVariableValue(&setting.Migrations.AllowedHostList, "")() + defer test.MockVariableValue(&setting.Migrations.BlockedHostList, "8.8.4.4")() + assert.NoError(t, IsMigrateURLAllowed("https://8.8.8.8/go-gitea/gitea.git", nonAdminUser)) + assert.Error(t, IsMigrateURLAllowed("https://8.8.4.4/go-gitea/gitea.git", nonAdminUser)) + assert.Error(t, IsMigrateURLAllowed("https://[64:ff9b::a9fe:a9fe]/go-gitea/gitea.git", nonAdminUser)) old := setting.ImportLocalPaths setting.ImportLocalPaths = false - err = IsMigrateURLAllowed("/home/foo/bar/goo", adminUser) - assert.Error(t, err) + assert.Error(t, IsMigrateURLAllowed("/home/foo/bar/goo", adminUser)) setting.ImportLocalPaths = true abs, err := filepath.Abs(".") assert.NoError(t, err) - err = IsMigrateURLAllowed(abs, adminUser) - assert.NoError(t, err) - - err = IsMigrateURLAllowed(abs, nonAdminUser) - assert.Error(t, err) + assert.NoError(t, IsMigrateURLAllowed(abs, adminUser)) + assert.Error(t, IsMigrateURLAllowed(abs, nonAdminUser)) nonAdminUser.AllowImportLocal = true - err = IsMigrateURLAllowed(abs, nonAdminUser) - assert.NoError(t, err) + assert.NoError(t, IsMigrateURLAllowed(abs, nonAdminUser)) setting.ImportLocalPaths = old } - -func TestAllowBlockList(t *testing.T) { - init := func(allow, block string, local bool) { - setting.Migrations.AllowedDomains = allow - setting.Migrations.BlockedDomains = block - setting.Migrations.AllowLocalNetworks = local - assert.NoError(t, Init()) - } - - // default, allow all external, block none, no local networks - init("", "", false) - assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4")})) - assert.Error(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("127.0.0.1")})) - - // allow all including local networks (it could lead to SSRF in production) - init("", "", true) - assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4")})) - assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("127.0.0.1")})) - - // allow wildcard, block some subdomains. every resolved address must still be allowed. - init("*.domain.com", "blocked.domain.com", false) - assert.NoError(t, checkByAllowBlockList("sub.domain.com", []net.IP{net.ParseIP("1.2.3.4")})) - assert.Error(t, checkByAllowBlockList("sub.domain.com", []net.IP{net.ParseIP("127.0.0.1")})) - assert.Error(t, checkByAllowBlockList("sub.domain.com", []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("127.0.0.1")})) - assert.Error(t, checkByAllowBlockList("blocked.domain.com", []net.IP{net.ParseIP("1.2.3.4")})) - assert.Error(t, checkByAllowBlockList("sub.other.com", []net.IP{net.ParseIP("1.2.3.4")})) - - // allow wildcard still follows the local network policy for resolved addresses. - init("*", "", false) - assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4")})) - assert.Error(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("127.0.0.1")})) - assert.Error(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("127.0.0.1")})) - - // local network can still be blocked - init("*", "127.0.0.*", false) - assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4")})) - assert.Error(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("127.0.0.1")})) - - // reset to allow local networks (mock servers use 127.0.0.1) - init("", "", true) -} diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index 54ad5b8fbb0..596ec8c5373 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -88,10 +88,11 @@ func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, passwo client: &http.Client{ Transport: roundTripperFunc( func(req *http.Request) (*http.Response, error) { + req = req.Clone(ctx) if username != "" && password != "" { req.SetBasicAuth(username, password) } - return httpTransport.RoundTrip(req.WithContext(ctx)) + return httpTransport.RoundTrip(req) }), }, userMap: make(map[int64]*onedevUser), diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index f026bf19fa4..6fb07a8ea33 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -18,7 +18,6 @@ import ( "gitea.dev/modules/lfs" "gitea.dev/modules/log" "gitea.dev/modules/process" - "gitea.dev/modules/proxy" repo_module "gitea.dev/modules/repository" "gitea.dev/modules/setting" "gitea.dev/modules/timeutil" @@ -69,9 +68,7 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error } func pruneBrokenReferences(ctx context.Context, m *repo_model.Mirror, repoLogName string, gitRepo git.RepositoryFacade, timeout time.Duration) error { - cmd := gitcmd.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout) - git.HandleGitCmdHTTPRedirection(cmd, m.GetRemoteName()) - stdout, _, pruneErr := cmd.WithRepo(gitRepo).RunStdString(ctx) + stdout, _, pruneErr := gitcmd.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout).WithRepo(gitRepo).RunStdString(ctx) if pruneErr != nil { // sanitize the output, since it may contain the remote address, which may contain a password stderrMessage := util.SanitizeCredentialURLs(pruneErr.Stderr()) @@ -122,17 +119,15 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*repo_module.SyncResu return nil, false } } - envs := proxy.EnvWithProxy(remoteURL.URL) timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second // use fetch but not remote update because git fetch support --tags but remote update doesn't cmdFetch := func() *gitcmd.Command { cmd := gitcmd.NewCommand("fetch", "--tags") - git.HandleGitCmdHTTPRedirection(cmd, m.GetRemoteName()) if m.EnablePrune { cmd.AddArguments("--prune") } - return cmd.AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout).WithEnv(envs) + return cmd.AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout) } var err error @@ -209,9 +204,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*repo_module.SyncResu } cmdRemoteUpdatePrune := func() *gitcmd.Command { - cmd := gitcmd.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout).WithEnv(envs) - git.HandleGitCmdHTTPRedirection(cmd, m.GetRemoteName()) - return cmd + return gitcmd.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout) } if repo_service.HasWiki(ctx, m.Repo) { diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 7b953862f52..3ce4d72d500 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -18,7 +18,6 @@ import ( "gitea.dev/modules/lfs" "gitea.dev/modules/log" "gitea.dev/modules/process" - "gitea.dev/modules/proxy" "gitea.dev/modules/repository" "gitea.dev/modules/setting" "gitea.dev/modules/timeutil" @@ -156,13 +155,11 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { log.Trace("Pushing mirror %d repo %s to remote %s", m.ID, storageRepo.LogString(), m.RemoteName) - envs := proxy.EnvWithProxy(remoteURL.URL) if err := git.PushToExternal(ctx, storageRepo, git.PushOptions{ Remote: m.RemoteName, Force: true, Mirror: true, Timeout: timeout, - Env: envs, }); err != nil { return fmt.Errorf("PushToExternal failed: %w", err) } diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index fc674f7ca6f..83387c2e5bc 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -16,17 +16,14 @@ import ( "net/http" "net/url" "strings" - "sync" "time" user_model "gitea.dev/models/user" webhook_model "gitea.dev/models/webhook" - "gitea.dev/modules/glob" + "gitea.dev/modules/egress" "gitea.dev/modules/graceful" - "gitea.dev/modules/hostmatcher" "gitea.dev/modules/log" "gitea.dev/modules/process" - "gitea.dev/modules/proxy" "gitea.dev/modules/queue" "gitea.dev/modules/setting" "gitea.dev/modules/timeutil" @@ -271,52 +268,15 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error { return nil } -var ( - webhookHTTPClient *http.Client - once sync.Once - hostMatchers []glob.Glob -) - -func webhookProxy(allowList *hostmatcher.HostMatchList) func(req *http.Request) (*url.URL, error) { - if setting.Webhook.ProxyURL == "" { - return proxy.Proxy() - } - - once.Do(func() { - for _, h := range setting.Webhook.ProxyHosts { - if g, err := glob.Compile(h); err == nil { - hostMatchers = append(hostMatchers, g) - } else { - log.Error("glob.Compile %s failed: %v", h, err) - } - } - }) - - return func(req *http.Request) (*url.URL, error) { - for _, v := range hostMatchers { - if v.Match(req.URL.Host) { - if !allowList.MatchHostName(req.URL.Host) { - return nil, fmt.Errorf("webhook can only call allowed HTTP servers (check your %s setting), deny '%s'", allowList.SettingKeyHint, req.URL.Host) - } - return http.ProxyURL(setting.Webhook.ProxyURLFixed)(req) - } - } - return http.ProxyFromEnvironment(req) - } -} +var webhookHTTPClient *http.Client // Init starts the hooks delivery thread func Init() error { timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second - allowedHostMatcher := hostmatcher.ParseHostMatchList("security.ALLOWED_HOST_LIST", setting.Webhook.AllowedHostList) - // NewHTTPTransport enforces the allow-list on direct connections; when webhookProxy routes a request - // through a configured proxy, restricting the proxied target is the proxy server's responsibility. - webhookHTTPClient = &http.Client{ - Timeout: timeout, - Transport: hostmatcher.NewHTTPTransport("webhook", allowedHostMatcher, nil, webhookProxy(allowedHostMatcher), setting.Webhook.ProxyURLFixed, - &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}), - } + transport := egress.NewWebhookPolicy().NewHTTPTransport() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify} + webhookHTTPClient = &http.Client{Timeout: timeout, Transport: transport} hookQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "webhook_sender", handler) if hookQueue == nil { diff --git a/services/webhook/deliver_test.go b/services/webhook/deliver_test.go index 83d55072fc4..cf954bbfc44 100644 --- a/services/webhook/deliver_test.go +++ b/services/webhook/deliver_test.go @@ -7,82 +7,18 @@ import ( "io" "net/http" "net/http/httptest" - "net/url" "strings" "testing" "time" "gitea.dev/models/unittest" webhook_model "gitea.dev/models/webhook" - "gitea.dev/modules/hostmatcher" - "gitea.dev/modules/setting" "gitea.dev/modules/util" webhook_module "gitea.dev/modules/webhook" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -func TestWebhookProxy(t *testing.T) { - oldWebhook := setting.Webhook - t.Cleanup(func() { - setting.Webhook = oldWebhook - }) - - setting.Webhook.ProxyURL = "http://localhost:8080" - setting.Webhook.ProxyURLFixed, _ = url.Parse(setting.Webhook.ProxyURL) - setting.Webhook.ProxyHosts = []string{"*.discordapp.com", "discordapp.com"} - - allowedHostMatcher := hostmatcher.ParseHostMatchList("webhook.ALLOWED_HOST_LIST", "discordapp.com,s.discordapp.com") - - tests := []struct { - req string - want string - wantErr bool - }{ - { - req: "https://discordapp.com/api/webhooks/xxxxxxxxx/xxxxxxxxxxxxxxxxxxx", - want: "http://localhost:8080", - wantErr: false, - }, - { - req: "http://s.discordapp.com/assets/xxxxxx", - want: "http://localhost:8080", - wantErr: false, - }, - { - req: "http://github.com/a/b", - want: "", - wantErr: false, - }, - { - req: "http://www.discordapp.com/assets/xxxxxx", - want: "", - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.req, func(t *testing.T) { - req, err := http.NewRequest(http.MethodPost, tt.req, nil) - require.NoError(t, err) - - u, err := webhookProxy(allowedHostMatcher)(req) - if tt.wantErr { - assert.Error(t, err) - return - } - - assert.NoError(t, err) - - got := "" - if u != nil { - got = u.String() - } - assert.Equal(t, tt.want, got) - }) - } -} - func TestWebhookDeliverAuthorizationHeader(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) diff --git a/services/webhook/main_test.go b/services/webhook/main_test.go index 9bed4df130d..1f584883c20 100644 --- a/services/webhook/main_test.go +++ b/services/webhook/main_test.go @@ -7,7 +7,6 @@ import ( "testing" "gitea.dev/models/unittest" - "gitea.dev/modules/hostmatcher" "gitea.dev/modules/setting" _ "gitea.dev/models" @@ -17,9 +16,7 @@ import ( func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ SetUp: func() error { - // for tests, allow only loopback IPs. This must run after the test config is loaded (which - // resets the shared Security.AllowedHostList) and before Init() builds the delivery client. - setting.Security.AllowedHostList = hostmatcher.MatchBuiltinLoopback + setting.Webhook.AllowedHostList = "loopback:*" setting.LoadQueueSettings() return Init() }, diff --git a/tests/integration/actions_schedule_test.go b/tests/integration/actions_schedule_test.go index 67e1829c0d6..230a8a41175 100644 --- a/tests/integration/actions_schedule_test.go +++ b/tests/integration/actions_schedule_test.go @@ -18,16 +18,12 @@ import ( "gitea.dev/models/unittest" user_model "gitea.dev/models/user" "gitea.dev/modules/migration" - "gitea.dev/modules/setting" api "gitea.dev/modules/structs" - "gitea.dev/modules/test" - migrations_service "gitea.dev/services/migrations" mirror_service "gitea.dev/services/mirror" repo_service "gitea.dev/services/repository" files_service "gitea.dev/services/repository/files" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestScheduleUpdate(t *testing.T) { @@ -148,11 +144,6 @@ jobs: func testScheduleUpdateMirrorSync(t *testing.T) { doTestScheduleUpdate(t, func(t *testing.T, u *url.URL, testContext APITestContext, user *user_model.User, repo *repo_model.Repository) (commitID, expectedSpec string) { - // the mirror sync re-validates the remote URL, which rejects the local test server unless local - // networks are allowed; migrations.Init rebuilds the host allow-list from the setting - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, true)() - require.NoError(t, migrations_service.Init()) - // create mirror repo opts := migration.MigrateOptions{ RepoName: "actions-schedule-mirror", diff --git a/tests/integration/api_repo_edit_test.go b/tests/integration/api_repo_edit_test.go index d60eb3c5752..d2c29a40f94 100644 --- a/tests/integration/api_repo_edit_test.go +++ b/tests/integration/api_repo_edit_test.go @@ -16,10 +16,7 @@ import ( "gitea.dev/models/unittest" user_model "gitea.dev/models/user" "gitea.dev/modules/git" - "gitea.dev/modules/setting" api "gitea.dev/modules/structs" - "gitea.dev/modules/test" - "gitea.dev/services/migrations" mirror_service "gitea.dev/services/mirror" "gitea.dev/tests" @@ -462,11 +459,7 @@ func TestAPIRepoEdit(t *testing.T) { mirror := unittest.AssertExistsAndLoadBean(t, &repo_model.Mirror{RepoID: 5}) newPassword := "updated-password" - require.NoError(t, mirror_service.UpdateAddress(ctx, mirror, "https://existing-user:existing-password@example.com/user2/repo1.git")) - - defer migrations.Init() - defer test.MockVariableValue(&setting.Migrations.AllowedDomains, "*")() - _ = migrations.Init() + require.NoError(t, mirror_service.UpdateAddress(ctx, mirror, "https://existing-user:existing-password@127.0.0.1/user2/repo1.git")) req = NewRequestWithJSON(t, "PATCH", fmt.Sprintf("/api/v1/repos/%s/%s", mirrorRepo.OwnerName, mirrorRepo.Name), &api.EditRepoOption{ MirrorPassword: &newPassword, @@ -474,10 +467,10 @@ func TestAPIRepoEdit(t *testing.T) { MakeRequest(t, req, http.StatusOK) updatedMirror := unittest.AssertExistsAndLoadBean(t, &repo_model.Mirror{RepoID: mirrorRepo.ID}) - assert.Equal(t, "https://example.com/user2/repo1.git", updatedMirror.RemoteAddress) + assert.Equal(t, "https://127.0.0.1/user2/repo1.git", updatedMirror.RemoteAddress) updatedRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: mirrorRepo.ID}) - assert.Equal(t, "https://example.com/user2/repo1.git", updatedRepo.OriginalURL) + assert.Equal(t, "https://127.0.0.1/user2/repo1.git", updatedRepo.OriginalURL) remoteURL, err := git.ParseRemoteAddressURL(ctx, updatedRepo, updatedMirror.GetRemoteName()) require.NoError(t, err) @@ -490,7 +483,7 @@ func TestAPIRepoEdit(t *testing.T) { // Test updating mirror token without guessing a username token := "mirror-token-value" - require.NoError(t, mirror_service.UpdateAddress(ctx, mirror, "https://example.com/user2/repo1.git")) + require.NoError(t, mirror_service.UpdateAddress(ctx, mirror, "https://127.0.0.1/user2/repo1.git")) req = NewRequestWithJSON(t, "PATCH", fmt.Sprintf("/api/v1/repos/%s/%s", mirrorRepo.OwnerName, mirrorRepo.Name), &api.EditRepoOption{ MirrorToken: &token, @@ -498,10 +491,10 @@ func TestAPIRepoEdit(t *testing.T) { MakeRequest(t, req, http.StatusOK) updatedMirror = unittest.AssertExistsAndLoadBean(t, &repo_model.Mirror{RepoID: mirrorRepo.ID}) - assert.Equal(t, "https://example.com/user2/repo1.git", updatedMirror.RemoteAddress) + assert.Equal(t, "https://127.0.0.1/user2/repo1.git", updatedMirror.RemoteAddress) updatedRepo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: mirrorRepo.ID}) - assert.Equal(t, "https://example.com/user2/repo1.git", updatedRepo.OriginalURL) + assert.Equal(t, "https://127.0.0.1/user2/repo1.git", updatedRepo.OriginalURL) remoteURL, err = git.ParseRemoteAddressURL(ctx, updatedRepo, updatedMirror.GetRemoteName()) require.NoError(t, err) diff --git a/tests/integration/api_repo_lfs_migrate_test.go b/tests/integration/api_repo_lfs_migrate_test.go index 5a92d6dea69..2b131f82429 100644 --- a/tests/integration/api_repo_lfs_migrate_test.go +++ b/tests/integration/api_repo_lfs_migrate_test.go @@ -15,7 +15,6 @@ import ( "gitea.dev/modules/setting" api "gitea.dev/modules/structs" "gitea.dev/modules/test" - "gitea.dev/services/migrations" "gitea.dev/tests" "github.com/stretchr/testify/assert" @@ -24,8 +23,6 @@ import ( func TestAPIRepoLFSMigrateLocal(t *testing.T) { defer tests.PrepareTestEnv(t)() defer test.MockVariableValue(&setting.ImportLocalPaths, true)() - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, true)() - assert.NoError(t, migrations.Init()) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) session := loginUser(t, user.Name) diff --git a/tests/integration/api_repo_test.go b/tests/integration/api_repo_test.go index 47d8effaab4..794f42c7838 100644 --- a/tests/integration/api_repo_test.go +++ b/tests/integration/api_repo_test.go @@ -18,12 +18,10 @@ import ( "gitea.dev/modules/setting" api "gitea.dev/modules/structs" "gitea.dev/modules/test" - "gitea.dev/services/migrations" repo_service "gitea.dev/services/repository" "gitea.dev/tests" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestAPIUserReposNotLogin(t *testing.T) { @@ -357,9 +355,6 @@ func TestAPIRepoMigrate(t *testing.T) { cloneAddr := fmt.Sprintf("%s%s/%s.git", u.String(), repo1.OwnerName, repo1.Name) t.Run("Permitted", func(t *testing.T) { - // migrations.Init builds the host allowlist from AllowLocalNetworks, so set it first - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, true)() - require.NoError(t, migrations.Init()) for _, testCase := range []struct { ctxUserID, ownerID int64 repoName string @@ -383,15 +378,13 @@ func TestAPIRepoMigrate(t *testing.T) { }) t.Run("DisallowedHost", func(t *testing.T) { - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, false)() - require.NoError(t, migrations.Init()) - token := getTokenForLoggedInUser(t, loginUser(t, "user2"), auth_model.AccessTokenScopeWriteRepository) + defer test.MockVariableValue(&setting.Migrations.AllowedHostList, "")() for _, cloneURL := range []string{"https://localhost:3000/user/test_repo.git", "https://10.0.0.1/user/test_repo.git"} { req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate", &api.MigrateRepoOptions{ CloneAddr: cloneURL, RepoOwnerID: 3, RepoName: "private-ip", - }).AddTokenAuth(token) + }).AddBasicAuth("user2") resp := MakeRequest(t, req, http.StatusUnprocessableEntity) assert.Equal(t, "You can not import from disallowed hosts.", DecodeJSON(t, resp, map[string]string{})["message"]) } diff --git a/tests/integration/dump_restore_test.go b/tests/integration/dump_restore_test.go index 92e0ed4664f..f38d38c10ea 100644 --- a/tests/integration/dump_restore_test.go +++ b/tests/integration/dump_restore_test.go @@ -31,8 +31,6 @@ func TestDumpRestore(t *testing.T) { onGiteaRun(t, func(t *testing.T, u *url.URL) { // Gitea SDK (go-sdk) need to parse the AppVer from server response, so we must set it to a valid version string. defer test.MockVariableValue(&setting.AppVer, "1.16.0")() - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, true)() - assert.NoError(t, migrations.Init()) reponame := "repo1" diff --git a/tests/integration/migrate_test.go b/tests/integration/migrate_test.go index a99ae1b94d9..50390cc7faa 100644 --- a/tests/integration/migrate_test.go +++ b/tests/integration/migrate_test.go @@ -61,9 +61,7 @@ func TestMigrateLocalPath(t *testing.T) { func TestMigrateGiteaForm(t *testing.T) { onGiteaRun(t, func(t *testing.T, u *url.URL) { // Gitea SDK (go-sdk) need to parse the AppVer from server response, so we must set it to a valid version string. - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, true)() defer test.MockVariableValue(&setting.AppVer, "1.16.0")() - assert.NoError(t, migrations.Init()) ownerName := "user2" repoName := "repo1" @@ -222,8 +220,6 @@ done func Test_MigrateFromGiteaToGitea(t *testing.T) { defer tests.PrepareTestEnv(t)() - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, true)() - assert.NoError(t, migrations.Init()) mockServer := setupGiteaMockServer(t) diff --git a/tests/integration/mirror_pull_test.go b/tests/integration/mirror_pull_test.go index b0c18d61230..9f80a3b1e49 100644 --- a/tests/integration/mirror_pull_test.go +++ b/tests/integration/mirror_pull_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "slices" + "strings" "sync/atomic" "testing" @@ -20,7 +21,6 @@ import ( "gitea.dev/modules/migration" "gitea.dev/modules/setting" "gitea.dev/modules/test" - migrations "gitea.dev/services/migrations" mirror_service "gitea.dev/services/mirror" release_service "gitea.dev/services/release" repo_service "gitea.dev/services/repository" @@ -139,40 +139,19 @@ func TestMirrorPullSSRFRevalidation(t *testing.T) { defer tests.PrepareTestEnv(t)() ctx := t.Context() - user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - repoPath := gitrepo.RepoLocalPath(repo) - - // an "internal" server that records whether it was reached - var reached atomic.Bool - internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - reached.Store(true) - w.WriteHeader(http.StatusNotFound) + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + http.Redirect(w, r, "http://"+strings.Replace(r.Host, "127.0.0.1", "0.0.0.0", 1)+r.URL.RequestURI(), http.StatusFound) })) - defer internal.Close() + defer server.Close() - mirrorRepo, err := repo_service.CreateRepositoryDirectly(ctx, user, user, repo_service.CreateRepoOptions{ - Name: "ssrf_mirror", - IsMirror: true, - Status: repo_model.RepositoryBeingMigrated, - }, false) - require.NoError(t, err) - _, err = repo_service.MigrateRepositoryGitData(ctx, user, mirrorRepo, migration.MigrateOptions{ - RepoName: "ssrf_mirror", - Mirror: true, - CloneAddr: repoPath, - }, nil) - require.NoError(t, err) + mirror := unittest.AssertExistsAndLoadBean(t, &repo_model.Mirror{RepoID: 5}) + require.NoError(t, mirror_service.UpdateAddress(ctx, mirror, server.URL+"/repo.git")) + assert.False(t, mirror_service.SyncPullMirror(ctx, mirror.RepoID)) + assert.EqualValues(t, 1, requests.Load(), "the git proxy must deny the redirect to a reserved address") - mirror, err := repo_model.GetMirrorByRepoID(ctx, mirrorRepo.ID) - require.NoError(t, err) - - // repoint the mirror at the loopback server, which is disallowed once local networks are off - require.NoError(t, mirror_service.UpdateAddress(ctx, mirror, internal.URL+"/repo.git")) - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, false)() - require.NoError(t, migrations.Init()) - t.Cleanup(func() { _ = migrations.Init() }) - - assert.False(t, mirror_service.SyncPullMirror(ctx, mirrorRepo.ID)) - assert.False(t, reached.Load(), "the disallowed internal remote must not be reached") + defer test.MockVariableValue(&setting.Migrations.AllowedHostList, "")() + assert.False(t, mirror_service.SyncPullMirror(ctx, mirror.RepoID)) + assert.EqualValues(t, 1, requests.Load(), "the disallowed internal remote must not be reached") } diff --git a/tests/integration/mirror_push_test.go b/tests/integration/mirror_push_test.go index fb15cb9189f..7defe9b94b9 100644 --- a/tests/integration/mirror_push_test.go +++ b/tests/integration/mirror_push_test.go @@ -18,7 +18,6 @@ import ( "gitea.dev/modules/git" "gitea.dev/modules/setting" "gitea.dev/modules/test" - "gitea.dev/services/migrations" mirror_service "gitea.dev/services/mirror" repo_service "gitea.dev/services/repository" wiki_service "gitea.dev/services/wiki" @@ -36,9 +35,6 @@ func TestMirrorPushWikiDefaultBranchMismatch(t *testing.T) { } func testMirrorPush(t *testing.T, u *url.URL) { - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, true)() - assert.NoError(t, migrations.Init()) - _ = db.TruncateBeans(t.Context(), &repo_model.PushMirror{}) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) srcRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) @@ -76,9 +72,7 @@ func testMirrorPush(t *testing.T, u *url.URL) { assert.Equal(t, srcCommit.ID, mirrorCommit.ID) - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, false)() // the remote is re-checked every sync, not just when added - assert.NoError(t, migrations.Init()) - t.Cleanup(func() { _ = migrations.Init() }) + defer test.MockVariableValue(&setting.Migrations.AllowedHostList, "")() assert.False(t, mirror_service.SyncPushMirror(t.Context(), mirrors[0].ID)) // Cleanup @@ -89,9 +83,6 @@ func testMirrorPush(t *testing.T, u *url.URL) { } func testMirrorPushWikiDefaultBranchMismatch(t *testing.T, u *url.URL) { - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, true)() - assert.NoError(t, migrations.Init()) - _ = db.TruncateBeans(t.Context(), &repo_model.PushMirror{}) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) srcRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) @@ -160,8 +151,6 @@ func doUpdatePushMirror(t *testing.T, session *TestSession, owner, repo string, func TestRepoSettingPushMirrorUpdate(t *testing.T) { defer tests.PrepareTestEnv(t)() - defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, true)() - assert.NoError(t, migrations.Init()) session := loginUser(t, "user2") repo2 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}) diff --git a/tests/mssql.ini.tmpl b/tests/mssql.ini.tmpl index aba67157df2..2b6e80ef0e0 100644 --- a/tests/mssql.ini.tmpl +++ b/tests/mssql.ini.tmpl @@ -50,6 +50,9 @@ ENABLED = true PROTOCOL = dummy FROM = mssql-integration-test@gitea.io +[migrations] +ALLOWED_HOST_LIST = private, loopback + [service] REGISTER_EMAIL_CONFIRM = false REGISTER_MANUAL_CONFIRM = false diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index 4572cfd724d..96739da5b2f 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -53,6 +53,9 @@ ENABLED = true PROTOCOL = dummy FROM = mysql-integration-test@gitea.io +[migrations] +ALLOWED_HOST_LIST = private, loopback + [service] REGISTER_EMAIL_CONFIRM = false REGISTER_MANUAL_CONFIRM = false diff --git a/tests/pgsql.ini.tmpl b/tests/pgsql.ini.tmpl index 5e15ca7ba6b..e71f5ce9ca5 100644 --- a/tests/pgsql.ini.tmpl +++ b/tests/pgsql.ini.tmpl @@ -51,6 +51,9 @@ ENABLED = true PROTOCOL = dummy FROM = pgsql-integration-test@gitea.io +[migrations] +ALLOWED_HOST_LIST = private, loopback + [service] REGISTER_EMAIL_CONFIRM = false REGISTER_MANUAL_CONFIRM = false diff --git a/tests/sqlite.ini.tmpl b/tests/sqlite.ini.tmpl index 95a1df283fa..47c85f2e936 100644 --- a/tests/sqlite.ini.tmpl +++ b/tests/sqlite.ini.tmpl @@ -47,6 +47,9 @@ ENABLED = true PROTOCOL = dummy FROM = sqlite-integration-test@gitea.io +[migrations] +ALLOWED_HOST_LIST = private, loopback + [service] REGISTER_EMAIL_CONFIRM = false REGISTER_MANUAL_CONFIRM = false