jwt_authn retry policy not working

Title: jwt_authn retry policy not working

Description:

While using the envoy.filters.http.jwt_authn HTTP filter, if the JWKS endpoint is not working, Envoy retries indefinitely to the JWKS endpoint, even when a retry policy is configured.

Repro steps:

Use the attached envoy configuration(it has a wrong jwks url). You can run this in docker using the following commands

1. run the envoy proxy

docker network create envoy-net

docker run --rm \
  --name envoy \
  -p 10000:10000 \
  -p 9901:9901 \
  --network envoy-net \
  -v "$(pwd)/envoy.yaml":/etc/envoy/envoy.yaml \
  envoyproxy/envoy:v1.30.1

2. Observe the logs and you will see indefinite logs like this [warning][jwt] [source/extensions/filters/http/jwt_authn/jwks_async_fetcher.cc:115] Jwks async fetching url=https://raw.githubusercontent.com/envoyproxy/gateway/main/examples/kubernetes/jwt/jwkss.json: failed

Config:

static_resources:
  listeners:
    - name: listener_0
      address:
        socket_address:
          address: 0.0.0.0
          port_value: 10000
      filter_chains:
        - filters:
            - name: envoy.filters.network.http_connection_manager
              typed_config:
                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                stat_prefix: ingress_http
                route_config:
                  name: local_route
                  virtual_hosts:
                    - name: local_service
                      domains: ["*"]
                      routes:
                        - match:
                            prefix: "/"
                          route:
                            cluster: httpbin_local
                http_filters:
                  - name: envoy.filters.http.jwt_authn
                    typed_config:     
                      "@type": type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.JwtAuthentication
                      providers:
                        httproute/default/backend/rule/0/match/0/www_example_com/example:
                          remote_jwks:
                            http_uri:
                              uri: https://raw.githubusercontent.com/envoyproxy/gateway/main/examples/kubernetes/jwt/jwkss.json
                              cluster: raw_githubusercontent_com_443
                              timeout: 1s
                            cache_duration: 300s
                            async_fetch: {}
                            retry_policy:
                              num_retries: 3
                              retry_on: connect-failure,refused-stream,unavailable,cancelled,retriable-status-codes
                          forward: true
                          payload_in_metadata: example
                          normalize_payload_in_metadata:
                            space_delimited_claims:
                            - scope
                      requirement_map:
                        httproute/default/backend/rule/0/match/0/www_example_com:
                          provider_name: httproute/default/backend/rule/0/match/0/www_example_com/example

                  - name: envoy.filters.http.router
                    typed_config:
                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
  clusters:
    - name: some_service
      connect_timeout: 0.25s
      type: LOGICAL_DNS
      lb_policy: ROUND_ROBIN
      load_assignment:
        cluster_name: some_service
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: jsonplaceholder.typicode.com
                      port_value: 80
    - name: httpbin_local
      connect_timeout: 0.25s
      type: STRICT_DNS
      lb_policy: ROUND_ROBIN
      load_assignment:
        cluster_name: httpbin_local
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: httpbin
                      port_value: 80
    - name: raw_githubusercontent_com_443
      type: STRICT_DNS
      connect_timeout: 10s
      per_connection_buffer_limit_bytes: 32768
      lb_policy: LEAST_REQUEST
      circuit_breakers:
        thresholds:
        - max_retries: 1024
      dns_refresh_rate: 30s
      dns_lookup_family: V4_PREFERRED
      transport_socket:
        name: envoy.transport_sockets.tls
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
          common_tls_context:
            validation_context:
              trusted_ca:
                filename: "/etc/ssl/certs/ca-certificates.crt"
          sni: raw.githubusercontent.com
      common_lb_config:
        locality_weighted_lb_config: {}
      load_assignment:
        cluster_name: raw_githubusercontent_com_443
        endpoints:
        - locality:
            region: raw_githubusercontent_com_443/backend/-1
          lb_endpoints:
          - endpoint:
              address:
                socket_address:
                  address: raw.githubusercontent.com
                  port_value: 443
            load_balancing_weight: 1
          load_balancing_weight: 1
      respect_dns_ttl: true
admin:
  access_log_path: /dev/null
  address:
    socket_address:
      address: 0.0.0.0
      port_value: 9901