diff --git a/internal/backends/proxy.go b/internal/backends/proxy.go
index 88dd8db..0244758 100644
--- a/internal/backends/proxy.go
+++ b/internal/backends/proxy.go
@@ -55,9 +55,6 @@ type proxySettings struct {
 func Proxy(route config.Route) router.Handler {
 	return func(sess ssh.Session, arg string) error {
 		user, _ := sshctx.GetUser(sess.Context())
-		if !route.Permissions.IsAllowed(user, "*") {
-			return router.ErrUnauthorized
-		}
 
 		var opts proxySettings
 		err := gocty.FromCtyValue(route.Settings, &opts)
@@ -81,22 +78,23 @@ func Proxy(route config.Route) router.Handler {
 			}
 		}
 
-		var matched bool
-		var addr, portstr string
+		matched := false
+		addr := arg
+		var portstr, pattern string
 		if opts.Host == nil {
 			hosts := ctyTupleToStrings(opts.Hosts)
 			if len(hosts) == 0 {
 				return errors.New("no host configuration provided")
 			}
-			
+
 			for _, hostPattern := range hosts {
-				addr, portstr, ok = strings.Cut(hostPattern, ":")
+				pattern, portstr, ok = strings.Cut(hostPattern, ":")
 				if !ok {
 					// addr is already set by the above statement, so just set the default port
 					portstr = "22"
 				}
 
-				matched, err = path.Match(addr, arg)
+				matched, err = path.Match(pattern, arg)
 				if err != nil {
 					return err
 				}
@@ -114,6 +112,10 @@ func Proxy(route config.Route) router.Handler {
 			}
 		}
 
+		if !route.Permissions.IsAllowed(user, addr) {
+			return router.ErrUnauthorized
+		}
+
 		if !matched {
 			return errors.New("provided argument doesn't match any host patterns in configuration")
 		}