Apollo Router Managed Federation Random Handshake Error

Help
1

I deployed Apollo Router helm chart version 1.27.0 configured with Managed Federation. I use Apollo Studio to automatically update the supergraph of Apollo Router. All subgraphs have an HTTPS url.

Most requests complete successfully, but randomly we get this error:

Fetch error: HTTP fetch failed from 'myapp-servers-graphql': error trying to connect: peer misbehaved: abbreviated handshake offered, but with varied cs

Sometimes it’s subgraph A, another time it’s subgraph B.

All subgraphs are .net 6 web api using Hot Chocolate GraphQL Server

Here is the complete log of one error:

{
    "timestamp": "2023-08-11T05:45:16.048556Z",
    "level": "ERROR",
    "fetch_error": "hyper::Error(Connect, Custom { kind: Other, error: Custom { kind: InvalidData, error: PeerMisbehavedError(\"abbreviated handshake offered, but with varied cs\") } })",
    "span": {
        "apollo.subgraph.name": "myapp-servers-graphql",
        "graphql.operation.name": "...",
        "http.route": "/api/servers/graphql/",
        "http.url": "https://api.myapp.com/api/servers/graphql/",
        "net.peer.name": "api.myapp.com",
        "net.peer.port": "443",
        "net.transport": "ip_tcp",
        "otel.kind": "CLIENT",
        "name": "subgraph_request"
    },
    "spans": [
        {
            "http.flavor": "HTTP/1.1",
            "http.method": "POST",
            "http.route": "/",
            "otel.kind": "SERVER",
            "name": "request"
        },
        {
            "apollo_private.http.request_headers": "{}",
            "client.name": "",
            "client.version": "",
            "http.flavor": "HTTP/1.1",
            "http.method": "POST",
            "http.route": "/",
            "otel.kind": "INTERNAL",
            "trace_id": "74b65fbb3e7b412ab1e78f69e0966228",
            "name": "router"
        },
        {
            "apollo_private.field_level_instrumentation_ratio": 0.01,
            "apollo_private.graphql.variables": "...",
            "graphql.document": "....",
            "graphql.operation.name": "",
            "otel.kind": "INTERNAL",
            "name": "supergraph"
        },
        {
            "graphql.operation.type": "query",
            "otel.kind": "INTERNAL",
            "name": "execution"
        },
        {
            "apollo.subgraph.name": "myapp-servers-graphql",
            "apollo_private.sent_time_offset": 391801,
            "otel.kind": "INTERNAL",
            "name": "fetch"
        },
        {
            "apollo.subgraph.name": "myapp-servers-graphql",
            "graphql.document": "...",
            "graphql.operation.name": "...",
            "otel.kind": "INTERNAL",
            "name": "subgraph"
        },
        {
            "apollo.subgraph.name": "myapp-servers-graphql",
            "graphql.operation.name": "...",
            "http.route": "/api/servers/graphql/",
            "http.url": "https://api.myapp.com/api/servers/graphql/",
            "net.peer.name": "api.myapp.com",
            "net.peer.port": "443",
            "net.transport": "ip_tcp",
            "otel.kind": "CLIENT",
            "name": "subgraph_request"
        }
    ]
}