0.33.0-rc1 loses all incoming connections after ~15 mins

[ylempereur]

Checklist

• This is a bug report, not a question. Ask questions on discuss.ipfs.tech.
• I have searched on the issue tracker for my bug.
• I am running the latest kubo version or have an issue updating.

Installation method

dist.ipfs.tech or ipfs-update

Version

Kubo version: 0.33.0-rc1
Repo version: 16
System version: amd64/darwin
Golang version: go1.23.4

Config

{
	"API": {
		"HTTPHeaders": {
			"Access-Control-Allow-Methods": [
				"PUT",
				"POST"
			],
			"Access-Control-Allow-Origin": [
				"http://webui.ipfs.tech.ipns.localhost:8080",
				"http://webui.ipfs.io.ipns.localhost:8080",
				"http://localhost:3000",
				"http://127.0.0.1:5001",
				"https://webui.ipfs.tech",
				"https://webui.ipfs.io"
			]
		}
	},
	"Addresses": {
		"API": "/ip4/127.0.0.1/tcp/5001",
		"Announce": null,
		"AppendAnnounce": [
			"/ip4/174.26.14.111/tcp/4001",
			"/ip4/174.26.14.111/udp/4001/webrtc-direct",
			"/ip4/174.26.14.111/udp/4001/quic-v1",
			"/ip4/174.26.14.111/udp/4001/quic-v1/webtransport"
		],
		"Gateway": "/ip4/127.0.0.1/tcp/8080",
		"NoAnnounce": null,
		"Swarm": [
			"/ip4/127.0.0.1/tcp/4001",
			"/ip4/127.0.0.1/udp/4001/webrtc-direct",
			"/ip4/127.0.0.1/udp/4001/quic-v1",
			"/ip4/127.0.0.1/udp/4001/quic-v1/webtransport",
			"/ip4/192.168.2.9/tcp/4001",
			"/ip4/192.168.2.9/udp/4001/webrtc-direct",
			"/ip4/192.168.2.9/udp/4001/quic-v1",
			"/ip4/192.168.2.9/udp/4001/quic-v1/webtransport",
			"/ip6/::1/tcp/4001",
			"/ip6/::1/udp/4001/webrtc-direct",
			"/ip6/::1/udp/4001/quic-v1",
			"/ip6/::1/udp/4001/quic-v1/webtransport",
			"/ip6/2602:ae:1a0e:6f02:14b2:1aa8:65d9:6380/tcp/4001",
			"/ip6/2602:ae:1a0e:6f02:14b2:1aa8:65d9:6380/udp/4001/webrtc-direct",
			"/ip6/2602:ae:1a0e:6f02:14b2:1aa8:65d9:6380/udp/4001/quic-v1",
			"/ip6/2602:ae:1a0e:6f02:14b2:1aa8:65d9:6380/udp/4001/quic-v1/webtransport"
		]
	},
	"AutoNAT": {},
	"Bootstrap": [
		"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
		"/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
		"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
		"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
		"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
		"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt"
	],
	"DNS": {
		"Resolvers": null
	},
	"Datastore": {
		"BloomFilterSize": 0,
		"GCPeriod": "1h",
		"HashOnRead": false,
		"Spec": {
			"mounts": [
				{
					"child": {
						"path": "blocks",
						"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
						"sync": true,
						"type": "flatfs"
					},
					"mountpoint": "/blocks",
					"prefix": "flatfs.datastore",
					"type": "measure"
				},
				{
					"child": {
						"compression": "none",
						"path": "datastore",
						"type": "levelds"
					},
					"mountpoint": "/",
					"prefix": "leveldb.datastore",
					"type": "measure"
				}
			],
			"type": "mount"
		},
		"StorageGCWatermark": 90,
		"StorageMax": "10GB"
	},
	"Discovery": {
		"MDNS": {
			"Enabled": false,
			"Interval": 10
		}
	},
	"Experimental": {
		"FilestoreEnabled": false,
		"GraphsyncEnabled": false,
		"Libp2pStreamMounting": false,
		"OptimisticProvide": true,
		"OptimisticProvideJobsPoolSize": 120,
		"P2pHttpProxy": false,
		"StrategicProviding": false,
		"UrlstoreEnabled": false
	},
	"Gateway": {
		"APICommands": [],
		"DeserializedResponses": null,
		"DisableHTMLErrors": null,
		"ExposeRoutingAPI": null,
		"HTTPHeaders": {},
		"NoDNSLink": false,
		"NoFetch": false,
		"PathPrefixes": [],
		"PublicGateways": null,
		"RootRedirect": "",
		"Writable": false
	},
	"Identity": {
		"PeerID": "12D3KooWKuSzGoorvsBB8zdXCYVMEfybYKa7sDgXT3AkofAzX3FV"
	},
	"Import": {
		"CidVersion": null,
		"HashFunction": null,
		"UnixFSChunker": null,
		"UnixFSRawLeaves": null
	},
	"Internal": {},
	"Ipns": {
		"RecordLifetime": "",
		"RepublishPeriod": "",
		"ResolveCacheSize": 128
	},
	"Migration": {
		"DownloadSources": null,
		"Keep": ""
	},
	"Mounts": {
		"FuseAllowOther": false,
		"IPFS": "/ipfs",
		"IPNS": "/ipns"
	},
	"Peering": {
		"Peers": [
			{
				"Addrs": [],
				"ID": "12D3KooWRzGji3gKV6sqkT5XBPE2PfR7Pgg59Jyia5zLtx8PFymT"
			},
			{
				"Addrs": [],
				"ID": "12D3KooWGTqxSXzf7oFxdkJcCR7oMXWsx7YDU3viBSGZdTg7M8rk"
			},
			{
				"Addrs": [],
				"ID": "12D3KooWABurJiWrFLWMQXVStkxJbUduT5UNbKzRmnBdMwcHm5CD"
			}
		]
	},
	"Pinning": {
		"RemoteServices": {
			"Web3_Storage": {
				"API": {
					"Endpoint": "https://api.web3.storage"
				},
				"Policies": {
					"MFS": {
						"Enable": false,
						"PinName": "",
						"RepinInterval": ""
					}
				}
			},
			"pinata": {
				"API": {
					"Endpoint": "https://api.pinata.cloud/psa"
				},
				"Policies": {
					"MFS": {
						"Enable": false,
						"PinName": "",
						"RepinInterval": ""
					}
				}
			}
		}
	},
	"Plugins": {
		"Plugins": null
	},
	"Provider": {
		"Strategy": ""
	},
	"Pubsub": {
		"DisableSigning": false,
		"Router": ""
	},
	"Reprovider": {
		"Interval": "12h0m0s",
		"Strategy": "all"
	},
	"Routing": {
		"AcceleratedDHTClient": false,
		"Methods": null,
		"Routers": null
	},
	"Swarm": {
		"AddrFilters": [
			"/ip4/10.0.0.0/ipcidr/8",
			"/ip4/100.64.0.0/ipcidr/10",
			"/ip4/169.254.0.0/ipcidr/16",
			"/ip4/172.16.0.0/ipcidr/12",
			"/ip4/192.0.0.0/ipcidr/24",
			"/ip4/192.0.2.0/ipcidr/24",
			"/ip4/192.168.128.0/ipcidr/17",
			"/ip4/192.168.64.0/ipcidr/18",
			"/ip4/192.168.32.0/ipcidr/19",
			"/ip4/192.168.16.0/ipcidr/20",
			"/ip4/192.168.8.0/ipcidr/21",
			"/ip4/192.168.4.0/ipcidr/22",
			"/ip4/192.168.0.0/ipcidr/23",
			"/ip4/198.18.0.0/ipcidr/15",
			"/ip4/198.51.100.0/ipcidr/24",
			"/ip4/203.0.113.0/ipcidr/24",
			"/ip4/240.0.0.0/ipcidr/4",
			"/ip6/100::/ipcidr/64",
			"/ip6/2001:2::/ipcidr/48",
			"/ip6/2001:db8::/ipcidr/32",
			"/ip6/fe80::/ipcidr/10"
		],
		"ConnMgr": {},
		"DisableBandwidthMetrics": false,
		"DisableNatPortMap": true,
		"RelayClient": {},
		"RelayService": {},
		"ResourceMgr": {
			"Enabled": true,
			"Limits": {},
			"MaxMemory": "4GiB"
		},
		"Transports": {
			"Multiplexers": {},
			"Network": {},
			"Security": {}
		}
	},
	"Version": {}
}

Description

When switching from 0.32.1 to 0.33.0-rc1, the node initially behaves normally and has a lot of incoming connections:

➜  ~ ipfs swarm peers | wc -l                                         
     316
➜  ~ ipfs swarm peers --direction | grep '^/ip4/.\+\sinbound$' | wc -l
     101
➜  ~ ipfs swarm peers --direction | grep '^/ip6/.\+\sinbound$' | wc -l
      23

After about 15 mins, all incoming connections have been lost (the few remaining ones are due to peering):

➜  ~ ipfs swarm peers | wc -l                                         
      31
➜  ~ ipfs swarm peers --direction | grep '^/ip4/.\+\sinbound$' | wc -l
       1
➜  ~ ipfs swarm peers --direction | grep '^/ip6/.\+\sinbound$' | wc -l
       2

Switching back to 0.32.1 fixes the problem immediately.

A few observations:

• the node switches to announcing itself with relay connections, showing that it's aware of its state
• trying to reach the node either using nc 127.0.0.1 4001 from itself or nc 192.168.2.9 4001 from another node on the same local network fails to connect (works with 0.32.1)
• I initially suspected the macOS firewall not recognizing the new binary, but that's not it:

➜  ~ /usr/libexec/ApplicationFirewall/socketfilterfw --getappblocked /Users/yves/bin/ipfs
Incoming connection to /Users/yves/bin/ipfs is permitted.

• I uploaded a diag profile to IPFS at:
  ipns://yves.lempereur.name/files/ipfs-profile-2025-01-07T15_22_07-07_00.zip