A single transient metadata-server timeout permanently disables credentials for the life of the process

#!/usr/bin/env ruby
# frozen_string_literal: true

# Reproduction: google-cloud-env caches a *negative* metadata-server verdict for
# the life of the process. One probe that times out past the 60 s warmup pins
# @existence = :no permanently, and every later Application Default Credentials
# lookup in that process fails, even though the metadata server is healthy again
# milliseconds later.
#
# No GCP access required. A fake metadata server runs on localhost and
# GCE_METADATA_HOST points at it.
#
#   gem install google-cloud-env -v 2.4.0
#   ruby metadata_latch_repro.rb
#
# K_SERVICE / K_REVISION / K_CONFIGURATION are set only to get past the SMBIOS
# hardware gate in gce_check, which would otherwise decide "definitely not on
# GCP" before any HTTP happens. On a real GCE VM or Cloud Run revision they are
# unnecessary; they are not part of the defect.
#
# warmup_time is set to 0 so the run takes seconds instead of a minute. It is
# equivalent to any process that has been alive longer than the default 60 s
# warmup, which is every long-lived worker in production.

require "socket"

# A metadata server that answers correctly, or stalls past the client's request
# timeout when @mode is :blip.
class FakeMetadataServer
  attr_accessor :mode

  def initialize
    @mode = :healthy
    @server = TCPServer.new "127.0.0.1", 0
    @requests = 0
    @thread = Thread.new { accept_loop }
  end

  def address
    "127.0.0.1:#{@server.addr[1]}"
  end

  attr_reader :requests

  def stop
    @thread.kill
    @server.close
  end

  private

  def accept_loop
    loop do
      socket = @server.accept
      Thread.new { handle socket }
    end
  rescue IOError
    nil
  end

  def handle socket
    socket.gets until socket.gets.to_s.strip.empty?
    @requests += 1
    # A blip: the connection opens, but no response arrives before the client's
    # 500 ms request timeout. This is what a momentarily overloaded metadata
    # server looks like from the client side.
    sleep 1.0 if @mode == :blip
    socket.print "HTTP/1.1 200 OK\r\nMetadata-Flavor: Google\r\nContent-Length: 2\r\n\r\nok"
  rescue StandardError
    nil
  ensure
    socket.close rescue nil
  end
end

server = FakeMetadataServer.new
ENV["GCE_METADATA_HOST"] = server.address
ENV["K_SERVICE"] = ENV["K_REVISION"] = ENV["K_CONFIGURATION"] = "repro"

require "google/cloud/env"
require "google/cloud/env/version"

def fresh_env
  env = Google::Cloud::Env.new
  env.compute_metadata.warmup_time = 0
  env
end

def report label, value
  puts format("    %-46s %s", label, value)
end

puts "google-cloud-env #{Google::Cloud::Env::VERSION}, metadata host #{ENV['GCE_METADATA_HOST']}"
puts

# Control. Proves the HTTP probe is what decides existence here, so the runs
# below are not being answered by the SMBIOS gate or by a cached verdict.
puts "[control] healthy server"
env = fresh_env
report "metadata?", env.metadata?
report "existence", env.compute_metadata.existence_immediate
puts

# The defect. A single blip while the server is briefly unresponsive, then the
# server is healthy for every subsequent call — and the process never recovers.
puts "[defect] one blip, then a healthy server forever"
env = fresh_env
server.mode = :blip
report "metadata? during the blip", env.metadata?
report "existence", env.compute_metadata.existence_immediate
server.mode = :healthy
before = server.requests
3.times { env.metadata? }
report "metadata? after the blip (server healthy)", env.metadata?
report "existence", env.compute_metadata.existence_immediate
report "HTTP requests made by those 3 calls", server.requests - before
puts

# The warmup boundary. The identical blip, on a process still inside its warmup
# window, leaves existence :unconfirmed and recovers on the next call. Only the
# warmup comparison in post_update_existence separates recovery from permanent
# failure, so what breaks a worker is being alive for more than 60 seconds.
puts "[control] the same blip, inside the warmup window"
env = Google::Cloud::Env.new
server.mode = :blip
report "metadata? during the blip", env.metadata?
report "existence", env.compute_metadata.existence_immediate
server.mode = :healthy
before = server.requests
report "metadata? after the blip", env.metadata?
report "existence", env.compute_metadata.existence_immediate
report "HTTP requests made by that call", server.requests - before
puts

# The asymmetry. A confirmed verdict cached forever is sound; the process is on
# GCP and will stay there. A negative verdict cached forever is not: it encodes
# a 100 ms network timeout as a permanent fact about the environment.
puts "[asymmetry] confirmed first, then the same blip"
env = fresh_env
env.metadata?
report "existence after a healthy probe", env.compute_metadata.existence_immediate
server.mode = :blip
env.metadata?
server.mode = :healthy
report "existence after the blip", env.compute_metadata.existence_immediate
puts

server.stop

puts <<~SUMMARY
  Expected: the [defect] run recovers once the server answers again.
  Actual:   existence is :no, check_existence short-circuits on it, and the
            three later calls make zero HTTP requests. In a real process every
            Application Default Credentials lookup from that point on raises
            "Your credentials were not found", until the process restarts.
SUMMARY