Gateway does not run Unicode Normalization Forms leading to seemingly identical paths not resolving when using different non normalized strings

Checklist

• This is a bug report, not a question. Ask questions on discuss.ipfs.io.
• I have searched on the issue tracker for my bug.
• I am running the latest kubo version or have an issue updating.

Installation method

ipfs-desktop

Version

Kubo version: 0.25.0
Repo version: 15
System version: amd64/darwin
Golang version: go1.21.5

Config

{
  "API": {
    "HTTPHeaders": {}
  },
  "Addresses": {
    "API": "/ip4/127.0.0.1/tcp/5001",
    "Announce": [],
    "AppendAnnounce": [
      "/ip4/150.23.2.29/udp/12345/quic-v1"
    ],
    "Gateway": "/ip4/127.0.0.1/tcp/8080",
    "NoAnnounce": [
      "/ip4/10.0.0.0/ipcidr/8",
      "/ip4/127.0.0.0/ipcidr/8",
      "/ip4/100.64.0.0/ipcidr/10",
      "/ip4/169.254.0.0/ipcidr/16",
      "/ip4/172.16.0.0/ipcidr/12",
      "/ip4/192.0.0.0/ipcidr/24",
      "/ip4/192.0.2.0/ipcidr/24",
      "/ip4/192.168.0.0/ipcidr/16",
      "/ip4/198.18.0.0/ipcidr/15",
      "/ip4/198.51.100.0/ipcidr/24",
      "/ip4/203.0.113.0/ipcidr/24",
      "/ip4/240.0.0.0/ipcidr/4",
      "/ip6/100::/ipcidr/64",
      "/ip6/2001:2::/ipcidr/48",
      "/ip6/2001:db8::/ipcidr/32",
      "/ip6/fc00::/ipcidr/7",
      "/ip6/fe80::/ipcidr/10"
    ],
    "Swarm": [
      "/ip4/0.0.0.0/udp/12345/quic-v1",
      "/ip6/::/tcp/4001",
      "/ip4/0.0.0.0/udp/4001/quic-v1",
      "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport",
      "/ip6/::/udp/4001/quic-v1",
      "/ip6/::/udp/4001/quic-v1/webtransport"
    ]
  },
  "AutoNAT": {},
  "Bootstrap": [
    "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
    "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
    "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
    "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
    "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
    "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
  ],
  "DNS": {
    "Resolvers": {}
  },
  "Datastore": {
    "BloomFilterSize": 0,
    "GCPeriod": "1h",
    "HashOnRead": false,
    "Spec": {
      "mounts": [
        {
          "child": {
            "path": "blocks",
            "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
            "sync": true,
            "type": "flatfs"
          },
          "mountpoint": "/blocks",
          "prefix": "flatfs.datastore",
          "type": "measure"
        },
        {
          "child": {
            "compression": "none",
            "path": "datastore",
            "type": "levelds"
          },
          "mountpoint": "/",
          "prefix": "leveldb.datastore",
          "type": "measure"
        }
      ],
      "type": "mount"
    },
    "StorageGCWatermark": 90,
    "StorageMax": "10GB"
  },
  "Discovery": {
    "MDNS": {
      "Enabled": false
    }
  },
  "DontCheckOSXFUSE": true,
  "Experimental": {
    "FilestoreEnabled": true,
    "GraphsyncEnabled": false,
    "Libp2pStreamMounting": true,
    "OptimisticProvide": false,
    "OptimisticProvideJobsPoolSize": 0,
    "P2pHttpProxy": false,
    "StrategicProviding": false,
    "UrlstoreEnabled": false
  },
  "Gateway": {
    "APICommands": [],
    "DeserializedResponses": null,
    "DisableHTMLErrors": null,
    "ExposeRoutingAPI": null,
    "HTTPHeaders": {},
    "NoDNSLink": false,
    "NoFetch": false,
    "PathPrefixes": [],
    "PublicGateways": null,
    "RootRedirect": ""
  },
  "Identity": {
    "PeerID": "censored"
  },
  "Internal": {},
  "Ipns": {
    "RecordLifetime": "48h",
    "RepublishPeriod": "",
    "ResolveCacheSize": 128
  },
  "Migration": {
    "DownloadSources": [],
    "Keep": ""
  },
  "Mounts": {
    "FuseAllowOther": false,
    "IPFS": "/ipfs",
    "IPNS": "/ipns"
  },
  "Peering": {
    "Peers": [
      {
        "Addrs": [
          "/ip4/10.0.1.101/udp/4001/quic"
        ],
        "ID": "censored"
      }
    ]
  },
  "Pinning": {
    "RemoteServices": {}
  },
  "Plugins": {
    "Plugins": null
  },
  "Provider": {
    "Strategy": ""
  },
  "Pubsub": {
    "DisableSigning": false,
    "Router": ""
  },
  "Reprovider": {},
  "Routing": {
    "AcceleratedDHTClient": false,
    "Methods": null,
    "Routers": null
  },
  "Swarm": {
    "AddrFilters": [
      "/ip4/10.0.0.0/ipcidr/8",
      "/ip4/100.64.0.0/ipcidr/10",
      "/ip4/169.254.0.0/ipcidr/16",
      "/ip4/172.16.0.0/ipcidr/12",
      "/ip4/192.0.0.0/ipcidr/24",
      "/ip4/192.0.2.0/ipcidr/24",
      "/ip4/192.168.0.0/ipcidr/16",
      "/ip4/198.18.0.0/ipcidr/15",
      "/ip4/198.51.100.0/ipcidr/24",
      "/ip4/203.0.113.0/ipcidr/24",
      "/ip4/240.0.0.0/ipcidr/4",
      "/ip6/100::/ipcidr/64",
      "/ip6/2001:2::/ipcidr/48",
      "/ip6/2001:db8::/ipcidr/32",
      "/ip6/fc00::/ipcidr/7",
      "/ip6/fe80::/ipcidr/10"
    ],
    "ConnMgr": {
      "GracePeriod": "2m0s",
      "HighWater": 128,
      "LowWater": 64,
      "Type": "basic"
    },
    "DisableBandwidthMetrics": false,
    "DisableNatPortMap": true,
    "RelayClient": {},
    "RelayService": {},
    "ResourceMgr": {},
    "Transports": {
      "Multiplexers": {},
      "Network": {},
      "Security": {}
    }
  }
}

Description

Hello,

I'm using IPFS-Desktop, but that's not important.

I'll create some folder called "Test" and put some random files in it with these file names: A-5x01 Tíha 1.txt A-5x02 Tíha 2.txt A-5x03 Fenomén strachu.txt A-5x04 Rozklad anděla.txt A-5x05 Stěna ztracených duší.txt A-5x06 Časová smyčka.txt

When I add the "Test" folder to IPFS, they give me the Test folder's CID QmdmqdwE1ZWzPKJVWyDvLqNY2PLedaeLQfoKuJSViHGpie. Now I have the file path like this "/ipfs/QmdmqdwE1ZWzPKJVWyDvLqNY2PLedaeLQfoKuJSViHGpie/A-5x01 Tíha 1.txt" for each file.

If the file or path contains any characters like this ÁáÄäÉéĚěÍíÓóÔôÚúŮůÝýČčďťŇňŘřŠšŽž, the URL-encoded link can be represented in two ways: No URL-encoded path http://127.0.0.1:8080/ipfs/QmdmqdwE1ZWzPKJVWyDvLqNY2PLedaeLQfoKuJSViHGpie/A-5x01 Tíha 1.txt Url encoded by adding ́ symbol (0xCC81 in UTF8) after i. This url can be represented by http gateway server. http://127.0.0.1:8080/ipfs/QmdmqdwE1ZWzPKJVWyDvLqNY2PLedaeLQfoKuJSViHGpie/A-5x01%20Ti%CC%81ha%201.txt Url encoded by adding í symbol (0xC3AD in UTF8) to the url. This url can't be represented by http gateway server. http://127.0.0.1:8080/ipfs/QmdmqdwE1ZWzPKJVWyDvLqNY2PLedaeLQfoKuJSViHGpie/A-5x01%20T%c3%adha%201.txt

These special symbols are commonly used in the Czech and Slovak languages and are found in many files and folders.

I've tried both url-encoded formats on some random Apache web server and they can represent both links. I'm trying some apps that can download files from http paths, but they use the second encoding method and can't find the file on the http gateway.

Test files are there: https://ipfs.io/ipfs/QmdmqdwE1ZWzPKJVWyDvLqNY2PLedaeLQfoKuJSViHGpie/

I test this behavior on IPFS-Desktop 0.32.0 for Windows and MacOS, on kubo 0.25.0 for MacOS and also https://ipfs.io/ gateway.

I hope it will be useful.