This issue is for linkerd2/policy-controller where this lib is being used.
We are testing linkerd2 stable-2.12.0 and we see that policy-controller running in the Destination pod is not able to connect to the API-server and ends up with crash/restart loop.
Our current installation with linkerd2 stable-2.11.3 all is good with the policy-controller being able to access API server.
What we see is that policy-controller is not using the KUBERNETES_SERVICE_HOST
env variable to connect to the API-server. Its using kubernetes.default.svc
as the url to API-server.
Would it be possible to have kubert
honor KUBERNETES_SERVICE_HOST
and KUBERNETES_SERVICE_PORT
when communication to the API-server?
Logs from policy-controller
container:
{"timestamp":"2022-09-02T09:28:19.075636Z","level":"DEBUG","fields":{"service.ready":true,"message":"processing request"},"target":"tower::buffer::worker","spans":[{"name":"networkauthentications"}]}
{"timestamp":"2022-09-02T09:28:19.075645Z","level":"DEBUG","fields":{"service.ready":true,"message":"processing request"},"target":"tower::buffer::worker","spans":[{"name":"httproutes"}]}
{"timestamp":"2022-09-02T09:28:19.075706Z","level":"DEBUG","fields":{"message":"requesting"},"target":"kube_client::client::builder","spans":[{"name":"httproutes"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/httproutes?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.075820Z","level":"DEBUG","fields":{"message":"requesting"},"target":"kube_client::client::builder","spans":[{"name":"meshtlsauthentications"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/meshtlsauthentications?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.075699Z","level":"DEBUG","fields":{"message":"resolving host=\"kubernetes.default.svc\""},"target":"hyper::client::connect::dns"}
{"timestamp":"2022-09-02T09:28:19.077097Z","level":"DEBUG","fields":{"message":"requesting"},"target":"kube_client::client::builder","spans":[{"name":"serverauthorizations"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1beta1/serverauthorizations?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.077093Z","level":"DEBUG","fields":{"message":"requesting"},"target":"kube_client::client::builder","spans":[{"name":"networkauthentications"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/networkauthentications?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.078258Z","level":"DEBUG","fields":{"message":"resolving host=\"kubernetes.default.svc\""},"target":"hyper::client::connect::dns"}
{"timestamp":"2022-09-02T09:28:19.079550Z","level":"DEBUG","fields":{"message":"resolving host=\"kubernetes.default.svc\""},"target":"hyper::client::connect::dns"}
{"timestamp":"2022-09-02T09:28:19.078270Z","level":"DEBUG","fields":{"message":"resolving host=\"kubernetes.default.svc\""},"target":"hyper::client::connect::dns"}
{"timestamp":"2022-09-02T09:28:19.079722Z","level":"DEBUG","fields":{"message":"connecting to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"pods"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/api/v1/pods?&labelSelector=linkerd.io%2Fcontrol-plane-ns","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.079751Z","level":"DEBUG","fields":{"message":"connecting to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"authorizationpolicies"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/authorizationpolicies?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.078340Z","level":"DEBUG","fields":{"message":"resolving host=\"kubernetes.default.svc\""},"target":"hyper::client::connect::dns"}
{"timestamp":"2022-09-02T09:28:19.079558Z","level":"DEBUG","fields":{"message":"resolving host=\"kubernetes.default.svc\""},"target":"hyper::client::connect::dns"}
{"timestamp":"2022-09-02T09:28:19.081938Z","level":"DEBUG","fields":{"message":"connecting to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"serverauthorizations"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1beta1/serverauthorizations?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.082054Z","level":"DEBUG","fields":{"message":"connecting to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"networkauthentications"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/networkauthentications?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.082074Z","level":"DEBUG","fields":{"message":"connecting to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"httproutes"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/httproutes?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.083221Z","level":"DEBUG","fields":{"message":"connected to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"pods"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/api/v1/pods?&labelSelector=linkerd.io%2Fcontrol-plane-ns","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.083242Z","level":"DEBUG","fields":{"message":"connected to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"serverauthorizations"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1beta1/serverauthorizations?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.083344Z","level":"DEBUG","fields":{"message":"connected to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"authorizationpolicies"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/authorizationpolicies?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.083535Z","level":"DEBUG","fields":{"message":"connecting to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"servers"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1beta1/servers?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.083555Z","level":"DEBUG","fields":{"message":"connected to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"networkauthentications"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/networkauthentications?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.083571Z","level":"DEBUG","fields":{"message":"connected to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"httproutes"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/httproutes?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.083901Z","level":"DEBUG","fields":{"message":"connecting to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"meshtlsauthentications"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/meshtlsauthentications?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.085181Z","level":"DEBUG","fields":{"message":"connected to 10.2.0.1:443"},"target":"hyper::client::connect::http","spans":[{"name":"meshtlsauthentications"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1alpha1/meshtlsauthentications?","otel.kind":"client","otel.name":"list","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.085214Z","level":"ERROR","fields":{"message":"failed with error error trying to connect: unexpected EOF"},"target":"kube_client::client::builder","spans":[{"name":"serverauthorizations"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/apis/policy.linkerd.io/v1beta1/serverauthorizations?","otel.kind":"client","otel.name":"list","otel.status_code":"ERROR","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.085219Z","level":"ERROR","fields":{"message":"failed with error error trying to connect: unexpected EOF"},"target":"kube_client::client::builder","spans":[{"name":"pods"},{"http.method":"GET","http.url":"https://kubernetes.default.svc/api/v1/pods?&labelSelector=linkerd.io%2Fcontrol-plane-ns","otel.kind":"client","otel.name":"list","otel.status_code":"ERROR","name":"HTTP"}]}
{"timestamp":"2022-09-02T09:28:19.085257Z","level":"INFO","fields":{"message":"stream failed","error":"failed to perform initial object list: HyperError: error trying to connect: unexpected EOF"},"target":"kubert::errors","spans":[{"name":"serverauthorizations"}]}
{"timestamp":"2022-09-02T09:28:19.085283Z","level":"INFO","fields":{"message":"stream failed","error":"failed to perform initial object list: HyperError: error trying to connect: unexpected EOF"},"target":"kubert::errors","spans":[{"name":"pods"}]}