1. What kops version are you running? The command kops version, will display
this information.
Version 1.12.2
2. What Kubernetes version are you running? kubectl version will print the
version if a cluster is running or provide the Kubernetes version specified as
a kops flag.
Client Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.1", GitCommit:"4485c6f18cee9a5d3c3b4e523bd27972b1b53892", GitTreeState:"clean", BuildDate:"2019-07-18T14:25:20Z", GoVersion:"go1.12.7", Compiler:"gc", Platform:"darwin/amd64"}
Server Version: version.Info{Major:"1", Minor:"12", GitVersion:"v1.12.8", GitCommit:"a89f8c11a5f4f132503edbc4918c98518fd504e3", GitTreeState:"clean", BuildDate:"2019-04-23T04:41:47Z", GoVersion:"go1.10.8", Compiler:"gc", Platform:"linux/amd64"}
3. What cloud provider are you using?
aws
4. What commands did you run? What is the simplest way to reproduce this issue?
kops update cluster
kops rolling-update
5. What happened after the commands executed?
kube-controller-manager reports the following error while
Unable to sync caches for garbage collector controller
Failed to list <nil>: client: etcd cluster is unavailable or misconfigured; error #0: net/http: HTTP/1.x transport connection broken: malformed HTTP response "\x15\x03\x01\x00\x02\x02"
6. What did you expect to happen? Upgrade kubernetes
7. Please provide your cluster manifest. Execute
kops get --name my.example.com -o yaml to display your cluster manifest.
You may want to remove your cluster name and other sensitive information.
apiVersion: kops/v1alpha2
kind: Cluster
metadata:
creationTimestamp: 2019-05-14T16:15:12Z
name: k8s.runningenv.cloud
spec:
api:
dns: {}
loadBalancer:
sslCertificate: arn:aws:acm:ca-central-1:111111111111:certificate/ccxcccss112-ddw-ddwd-ddsds-ddsds3232
type: Public
authentication:
aws: {}
authorization:
rbac: {}
channel: stable
cloudProvider: aws
configBase: s3://dummy-data-store/k8s.runningenv.cloud
etcdClusters:
- etcdMembers:
- instanceGroup: master-ca-central-1a
name: a
name: main
- etcdMembers:
- instanceGroup: master-ca-central-1a
name: a
name: events
iam:
allowContainerRegistry: true
legacy: false
kubelet:
anonymousAuth: false
kubernetesApiAccess:
- 1.1.1.1/32
- 2.2.2.2/32
kubernetesVersion: 1.12.8
masterInternalName: api.internal.k8s.runningenv.cloud
masterPublicName: api.k8s.runningenv.cloud
networkCIDR: 172.20.0.0/16
networking:
amazonvpc: {}
nonMasqueradeCIDR: 172.20.0.0/16
sshAccess:
- 1.1.1.1/32
- 2.2.2.2/32
subnets:
- cidr: 172.20.32.0/19
name: ca-central-1a
type: Public
zone: ca-central-1a
- cidr: 172.20.64.0/19
name: ca-central-1b
type: Public
zone: ca-central-1b
topology:
dns:
type: Public
masters: public
nodes: public
---
apiVersion: kops/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: 2019-07-19T16:24:11Z
labels:
kops.k8s.io/cluster: k8s.runningenv.cloud
name: large_nodes
spec:
image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
machineType: t3.xlarge
maxSize: 2
minSize: 2
nodeLabels:
kops.k8s.io/instancegroup: large_nodes
role: Node
rootVolumeSize: 64
subnets:
- ca-central-1a
- ca-central-1b
---
apiVersion: kops/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: 2019-05-14T16:15:13Z
labels:
kops.k8s.io/cluster: k8s.runningenv.cloud
name: master-ca-central-1a
spec:
image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
machineType: c4.large
maxSize: 1
minSize: 1
nodeLabels:
kops.k8s.io/instancegroup: master-ca-central-1a
role: Master
subnets:
- ca-central-1a
---
apiVersion: kops/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: 2019-05-14T16:15:13Z
labels:
kops.k8s.io/cluster: k8s.runningenv.cloud
name: nodes
spec:
image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
machineType: t3.large
maxSize: 5
minSize: 5
nodeLabels:
kops.k8s.io/instancegroup: nodes
role: Node
subnets:
- ca-central-1a
- ca-central-1b
8. Please run the commands with most verbose logging by adding the -v 10 flag.
Paste the logs into this report, or in a gist and provide the gist link here.
Logs from kube-controller-manager
I0723 17:05:09.115034 1 shared_informer.go:119] stop requested
E0723 17:05:09.115219 1 controller_utils.go:1030] Unable to sync caches for garbage collector controller
E0723 17:05:09.115373 1 garbagecollector.go:233] timed out waiting for dependency graph builder sync during GC sync (attempt 300)
E0723 17:05:09.130184 1 reflector.go:125] k8s.io/kubernetes/pkg/controller/garbagecollector/graph_builder.go:124: Failed to list <nil>: client: etcd cluster is unavailable or misconfigured; error #0: net/http: HTTP/1.x transport connection broken: malformed HTTP response "\x15\x03\x01\x00\x02\x02"
E0723 17:05:09.144553 1 reflector.go:125] k8s.io/kubernetes/pkg/controller/garbagecollector/graph_builder.go:124: Failed to list <nil>: client: etcd cluster is unavailable or misconfigured; error #0: net/http: HTTP/1.x transport connection broken: malformed HTTP response "\x15\x03\x01\x00\x02\x02"
I0723 17:05:09.416248 1 garbagecollector.go:204] syncing garbage collector with updated resources from discovery (attempt 301): added: [{Group:auth.kope.io Version:v1alpha1 Resource:users} {Group:config.auth.kope.io Version:v1alpha1 Resource:authconfigurations} {Group:config.auth.kope.io Version:v1alpha1 Resource:authproviders}], removed: []
I0723 17:05:09.618069 1 controller_utils.go:1027] Waiting for caches to sync for garbage collector controller
E0723 17:05:10.133851 1 reflector.go:125] k8s.io/kubernetes/pkg/controller/garbagecollector/graph_builder.go:124: Failed to list <nil>: client: etcd cluster is unavailable or misconfigured; error #0: net/http: HTTP/1.x transport connection broken: malformed HTTP response "\x15\x03\x01\x00\x02\x02"
E0723 17:05:10.152706 1 reflector.go:125]
9. Anything else do we need to know? Tried connecting to etcd and this was the response
root@ip-172-20-48-95:/tmp/etcd-download-test# ./etcdctl --endpoints https://127.0.0.1:2380 --cert-file=/etc/kubernetes/pki/kube-apiserver/etcd-client.crt --key-file=/etc/kubernetes/pki/kube-apiserver/etcd-client.key --ca-file=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt --debug cluster-health
Cluster-Endpoints: https://127.0.0.1:2380
cURL Command: curl -X GET https://127.0.0.1:2380/v2/members
cluster may be unhealthy: failed to list members
Error: client: etcd cluster is unavailable or misconfigured; error #0: x509: certificate signed by unknown authority
error #0: x509: certificate signed by unknown authority
kubectl get cs response
I0723 13:16:52.688710 95389 request.go:947] Response Body: {"kind":"ComponentStatusList","apiVersion":"v1","metadata":{"selfLink":"/api/v1/componentstatuses"},"items":[{"metadata":{"name":"scheduler","selfLink":"/api/v1/componentstatuses/scheduler","creationTimestamp":null},"conditions":[{"type":"Healthy","status":"True","message":"ok"}]},{"metadata":{"name":"controller-manager","selfLink":"/api/v1/componentstatuses/controller-manager","creationTimestamp":null},"conditions":[{"type":"Healthy","status":"True","message":"ok"}]},{"metadata":{"name":"etcd-1","selfLink":"/api/v1/componentstatuses/etcd-1","creationTimestamp":null},"conditions":[{"type":"Healthy","status":"True","message":"{\"health\": \"true\"}"}]},{"metadata":{"name":"etcd-0","selfLink":"/api/v1/componentstatuses/etcd-0","creationTimestamp":null},"conditions":[{"type":"Healthy","status":"True","message":"{\"health\": \"true\"}"}]}]}
I0723 13:16:52.693112 95389 table_printer.go:43] Unable to decode server response into a Table. Falling back to hardcoded types: attempt to decode non-Table object into a v1beta1.Table
I0723 13:16:52.693491 95389 table_printer.go:43] Unable to decode server response into a Table. Falling back to hardcoded types: attempt to decode non-Table object into a v1beta1.Table
I0723 13:16:52.693534 95389 table_printer.go:43] Unable to decode server response into a Table. Falling back to hardcoded types: attempt to decode non-Table object into a v1beta1.Table
I0723 13:16:52.693556 95389 table_printer.go:43] Unable to decode server response into a Table. Falling back to hardcoded types: attempt to decode non-Table object into a v1beta1.Table
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-1 Healthy {"health": "true"}
etcd-0 Healthy {"health": "true"}
