area/securitybughelp wanted
Repository metrics
- Stars
- (27,997 stars)
- PR merge metrics
- (Avg merge 8d) (378 merged PRs in 30d)
Description
Description:
When Istio is configured to use the Secret Discovery Service, it configures Envoy as follows.
If the file /var/run/secrets/tokens/istio-token is missing or inaccessible, Envoy segfaults.
"tls_certificate_sds_secret_configs": [
{
"name": "default",
"sds_config": {
"api_config_source": {
"api_type": "GRPC",
"grpc_services": [
{
"google_grpc": {
"target_uri": "unix:/var/run/sds/uds_path",
"channel_credentials": {
"local_credentials": {}
},
"call_credentials": [
{
"from_plugin": {
"name": "envoy.grpc_credentials.file_based_metadata",
"typed_config": {
"@type": "type.googleapis.com/envoy.config.grpc_credential.v2alpha.FileBasedMetadataConfig",
"secret_data": {
"filename": "/var/run/secrets/tokens/istio-token"
},
"header_key": "istio_sds_credentials_header-bin"
}
}
}
],
"stat_prefix": "sdsstat",
"credentials_factory_name": "envoy.grpc_credentials.file_based_metadata"
}
}
]
}
}
}
]
Repro steps:
- Set up SDS similar to above.
- Remove
/var/run/secrets/tokens/istio-token
Logs:
2020-01-21T19:29:35.136910Z info FLAG: --applicationPorts="[]"
2020-01-21T19:29:35.136933Z info FLAG: --binaryPath="/usr/local/bin/envoy"
2020-01-21T19:29:35.136937Z info FLAG: --concurrency="2"
2020-01-21T19:29:35.136940Z info FLAG: --configPath="/etc/istio/proxy"
2020-01-21T19:29:35.136944Z info FLAG: --connectTimeout="10s"
2020-01-21T19:29:35.136947Z info FLAG: --controlPlaneAuthPolicy="NONE"
2020-01-21T19:29:35.136952Z info FLAG: --controlPlaneBootstrap="true"
2020-01-21T19:29:35.136955Z info FLAG: --customConfigFile=""
2020-01-21T19:29:35.136960Z info FLAG: --datadogAgentAddress=""
2020-01-21T19:29:35.136963Z info FLAG: --disableInternalTelemetry="false"
2020-01-21T19:29:35.136966Z info FLAG: --discoveryAddress="istio-pilot.istio-system:15010"
2020-01-21T19:29:35.136968Z info FLAG: --dnsRefreshRate="300s"
2020-01-21T19:29:35.136972Z info FLAG: --domain="service.svc.cluster.local"
2020-01-21T19:29:35.136977Z info FLAG: --drainDuration="45s"
2020-01-21T19:29:35.136980Z info FLAG: --envoyAccessLogService=""
2020-01-21T19:29:35.136983Z info FLAG: --envoyMetricsService=""
2020-01-21T19:29:35.136987Z info FLAG: --help="false"
2020-01-21T19:29:35.136990Z info FLAG: --id=""
2020-01-21T19:29:35.136993Z info FLAG: --ip=""
2020-01-21T19:29:35.136997Z info FLAG: --lightstepAccessToken=""
2020-01-21T19:29:35.137000Z info FLAG: --lightstepAddress=""
2020-01-21T19:29:35.137004Z info FLAG: --lightstepCacertPath=""
2020-01-21T19:29:35.137008Z info FLAG: --lightstepSecure="false"
2020-01-21T19:29:35.137011Z info FLAG: --log_as_json="false"
2020-01-21T19:29:35.137014Z info FLAG: --log_caller=""
2020-01-21T19:29:35.137018Z info FLAG: --log_output_level="default:info"
2020-01-21T19:29:35.137021Z info FLAG: --log_rotate=""
2020-01-21T19:29:35.137025Z info FLAG: --log_rotate_max_age="30"
2020-01-21T19:29:35.137036Z info FLAG: --log_rotate_max_backups="1000"
2020-01-21T19:29:35.137041Z info FLAG: --log_rotate_max_size="104857600"
2020-01-21T19:29:35.137045Z info FLAG: --log_stacktrace_level="default:none"
2020-01-21T19:29:35.137054Z info FLAG: --log_target="[stdout]"
2020-01-21T19:29:35.137062Z info FLAG: --mixerIdentity=""
2020-01-21T19:29:35.137066Z info FLAG: --parentShutdownDuration="1m0s"
2020-01-21T19:29:35.137069Z info FLAG: --pilotIdentity=""
2020-01-21T19:29:35.137074Z info FLAG: --proxyAdminPort="15000"
2020-01-21T19:29:35.137077Z info FLAG: --proxyComponentLogLevel="misc:error"
2020-01-21T19:29:35.137081Z info FLAG: --proxyLogLevel="warning"
2020-01-21T19:29:35.137090Z info FLAG: --serviceCluster="clustername"
2020-01-21T19:29:35.137093Z info FLAG: --serviceregistry="Kubernetes"
2020-01-21T19:29:35.137096Z info FLAG: --statsdUdpAddress=""
2020-01-21T19:29:35.137100Z info FLAG: --statusPort="15020"
2020-01-21T19:29:35.137103Z info FLAG: --templateFile=""
2020-01-21T19:29:35.137106Z info FLAG: --trust-domain=""
2020-01-21T19:29:35.137111Z info FLAG: --zipkinAddress="zipkin:80"
2020-01-21T19:29:35.137140Z info Version 1.4.2-35eb9dc7c6e78dac5bd8c3d142bc2a4601616932-Clean
2020-01-21T19:29:35.137267Z info Obtained private IP [10.16.2.16]
2020-01-21T19:29:35.137316Z info Proxy role: &model.Proxy{ClusterID:"", Type:"sidecar", IPAddresses:[]string{"10.16.2.16", "10.16.2.16"}, ID:"service-58f76459cd-4stp4", Locality:(*envoy_api_v2_core.Locality)(nil), DNSDomain:"service.svc.cluster.local", ConfigNamespace:"", Metadata:(*model.NodeMetadata)(nil), SidecarScope:(*model.SidecarScope)(nil), MergedGateway:(*model.MergedGateway)(nil), ServiceInstances:[]*model.ServiceInstance(nil), WorkloadLabels:labels.Collection(nil), IstioVersion:(*model.IstioVersion)(nil)}
2020-01-21T19:29:35.137325Z info PilotSAN []string(nil)
2020-01-21T19:29:35.137329Z info MixerSAN []string(nil)
2020-01-21T19:29:35.137762Z info Effective config: binaryPath: /usr/local/bin/envoy
concurrency: 2
configPath: /etc/istio/proxy
connectTimeout: 10s
discoveryAddress: istio-pilot.istio-system:15010
drainDuration: 45s
envoyAccessLogService: {}
envoyMetricsService: {}
parentShutdownDuration: 60s
proxyAdminPort: 15000
serviceCluster: service.namespace
statNameLength: 189
tracing:
zipkin:
address: zipkin:80
2020-01-21T19:29:35.137800Z info PilotSAN []string(nil)
2020-01-21T19:29:35.137813Z info Starting proxy agent
2020-01-21T19:29:35.137876Z info Received new config, creating new Envoy epoch 0
2020-01-21T19:29:35.137937Z warn watching /etc/certs encountered an error no such file or directory
2020-01-21T19:29:35.137945Z info Epoch 0 starting
2020-01-21T19:29:35.138522Z info Opening status port 15020
2020-01-21T19:29:35.141914Z info Envoy command: [-c /etc/istio/proxy/envoy-rev0.json --restart-epoch 0 --drain-time-s 45 --parent-shutdown-time-s 60 --service-cluster service --service-node sidecar~10.16.2.16~service-58f76459cd-4stp4.namespace~namespace.svc.cluster.local --max-obj-name-len 189 --local-address-ip-version v4 --log-format [Envoy (Epoch 0)] [%Y-%m-%d %T.%e][%t][%l][%n] %v -l warning --component-log-level misc:error --concurrency 2]
[Envoy (Epoch 0)] [2020-01-21 19:29:35.200][19][warning][config] [bazel-out/k8-opt/bin/external/envoy/source/common/config/_virtual_includes/grpc_stream_lib/common/config/grpc_stream.h:91] gRPC config stream closed: 14, no healthy upstream
[Envoy (Epoch 0)] [2020-01-21 19:29:35.200][19][warning][config] [bazel-out/k8-opt/bin/external/envoy/source/common/config/_virtual_includes/grpc_stream_lib/common/config/grpc_stream.h:54] Unable to establish new stream
[Envoy (Epoch 0)] [2020-01-21 19:29:37.065][19][warning][config] [external/envoy/source/common/config/grpc_mux_subscription_impl.cc:82] gRPC config for type.googleapis.com/envoy.api.v2.Cluster rejected: Error adding/updating cluster(s) outbound|9093||ajna-kafka.events.ajna.local.sfdc.net: unable to read file: /etc/identity/client/keys/client-key.pem
[Envoy (Epoch 0)] [2020-01-21 19:29:37.074][28][critical][main] [external/envoy/source/exe/terminate_handler.cc:13] std::terminate called! (possible uncaught exception, see trace)
[Envoy (Epoch 0)] [2020-01-21 19:29:37.074][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:70] Backtrace (use tools/stack_decode.py to get line numbers):
[Envoy (Epoch 0)] [2020-01-21 19:29:37.074][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:71] Envoy version: a7b2578494d79fe098f0e99282e0236946562fa0/1.12.0/Clean/RELEASE/BoringSSL
[Envoy (Epoch 0)] [2020-01-21 19:29:37.075][31][critical][main] [external/envoy/source/exe/terminate_handler.cc:13] std::terminate called! (possible uncaught exception, see trace)
[Envoy (Epoch 0)] [2020-01-21 19:29:37.075][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:70] Backtrace (use tools/stack_decode.py to get line numbers):
[Envoy (Epoch 0)] [2020-01-21 19:29:37.075][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:71] Envoy version: a7b2578494d79fe098f0e99282e0236946562fa0/1.12.0/Clean/RELEASE/BoringSSL
[Envoy (Epoch 0)] [2020-01-21 19:29:37.080][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #0: Envoy::TerminateHandler::logOnTerminate()::$_0::operator()() [0x5616fb1973ae]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.081][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #0: Envoy::TerminateHandler::logOnTerminate()::$_0::operator()() [0x5616fb1973ae]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.086][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:77] #1: [0x5616fb1972b9]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.087][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:77] #1: [0x5616fb1972b9]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.092][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #2: std::__terminate() [0x5616fb4e9d93]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.092][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #2: std::__terminate() [0x5616fb4e9d93]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.097][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #3: Envoy::Config::DataSource::read() [0x5616fad80b20]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.098][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #3: Envoy::Config::DataSource::read() [0x5616fad80b20]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.103][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #4: Envoy::Extensions::GrpcCredentials::FileBasedMetadata::FileBasedMetadataAuthenticator::GetMetadata() [0x5616f9ea5820]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.103][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #4: Envoy::Extensions::GrpcCredentials::FileBasedMetadata::FileBasedMetadataAuthenticator::GetMetadata() [0x5616f9ea5820]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.109][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #5: grpc::MetadataCredentialsPluginWrapper::InvokePlugin() [0x5616fad84508]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.109][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #5: grpc::MetadataCredentialsPluginWrapper::InvokePlugin() [0x5616fad84508]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.114][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #6: std::__1::__function::__func<>::operator()() [0x5616fad85393]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.115][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #6: std::__1::__function::__func<>::operator()() [0x5616fad85393]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.120][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #7: grpc::DynamicThreadPool::ThreadFunc() [0x5616fad8acb1]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.121][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #7: grpc::DynamicThreadPool::ThreadFunc() [0x5616fad8acb1]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.126][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #8: grpc::DynamicThreadPool::DynamicThread::ThreadFunc() [0x5616fad8aa05]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.126][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #8: grpc::DynamicThreadPool::DynamicThread::ThreadFunc() [0x5616fad8aa05]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.131][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #9: grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix()::{lambda()#1}::__invoke() [0x5616fae4fca3]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.131][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #10: start_thread [0x7fd05ee266db]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.131][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:83] Caught Aborted, suspect faulting address 0x53900000013
[Envoy (Epoch 0)] [2020-01-21 19:29:37.131][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:70] Backtrace (use tools/stack_decode.py to get line numbers):
[Envoy (Epoch 0)] [2020-01-21 19:29:37.131][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:71] Envoy version: a7b2578494d79fe098f0e99282e0236946562fa0/1.12.0/Clean/RELEASE/BoringSSL
[Envoy (Epoch 0)] [2020-01-21 19:29:37.132][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #0: __restore_rt [0x7fd05ee31890]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.132][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #9: grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix()::{lambda()#1}::__invoke() [0x5616fae4fca3]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.132][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #10: start_thread [0x7fd05ee266db]
2020-01-21T19:29:37.134846Z error Epoch 0 exited with error: signal: aborted
2020-01-21T19:29:37.134869Z info No more active epochs, terminating
Call Stack: I couldn't get stack_decode.py working on my mac. 😕 Here's what I had in the logs:
[Envoy (Epoch 0)] [2020-01-21 19:29:37.075][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:70] Backtrace (use tools/stack_decode.py to get line numbers):
[Envoy (Epoch 0)] [2020-01-21 19:29:37.075][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:71] Envoy version: a7b2578494d79fe098f0e99282e0236946562fa0/1.12.0/Clean/RELEASE/BoringSSL
[Envoy (Epoch 0)] [2020-01-21 19:29:37.080][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #0: Envoy::TerminateHandler::logOnTerminate()::$_0::operator()() [0x5616fb1973ae]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.086][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:77] #1: [0x5616fb1972b9]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.092][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #2: std::__terminate() [0x5616fb4e9d93]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.097][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #3: Envoy::Config::DataSource::read() [0x5616fad80b20]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.103][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #4: Envoy::Extensions::GrpcCredentials::FileBasedMetadata::FileBasedMetadataAuthenticator::GetMetadata() [0x5616f9ea5820]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.109][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #5: grpc::MetadataCredentialsPluginWrapper::InvokePlugin() [0x5616fad84508]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.115][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #6: std::__1::__function::__func<>::operator()() [0x5616fad85393]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.121][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #7: grpc::DynamicThreadPool::ThreadFunc() [0x5616fad8acb1]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.126][31][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #8: grpc::DynamicThreadPool::DynamicThread::ThreadFunc() [0x5616fad8aa05]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.131][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #9: grpc_core::(anonymous namespace)::ThreadInternalsPosix::ThreadInternalsPosix()::{lambda()#1}::__invoke() [0x5616fae4fca3]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.131][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #10: start_thread [0x7fd05ee266db]
[Envoy (Epoch 0)] [2020-01-21 19:29:37.131][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:83] Caught Aborted, suspect faulting address 0x53900000013
[Envoy (Epoch 0)] [2020-01-21 19:29:37.132][28][critical][backtrace] [bazel-out/k8-opt/bin/external/envoy/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:75] #0: __restore_rt [0x7fd05ee31890]