diff --git a/.jenkins/deploy-setup.yaml b/.jenkins/deploy-setup.yaml index 458625f55..7e2d46fb8 100644 --- a/.jenkins/deploy-setup.yaml +++ b/.jenkins/deploy-setup.yaml @@ -10,9 +10,11 @@ cluster: vlan: 612 addresses: - cidr-nodes: 10.1.0.0/24 + cidr-nodes: 10.1.0.0/23 cidr-pods: 172.18.0.0/16 cidr-services: 172.28.0.0/16 + external-ranges: + - 10.1.1.0/24 service-api: 172.28.0.1 service-dns: 172.28.0.2 diff --git a/deploy-chroot/setup.yaml.in b/deploy-chroot/setup.yaml.in index e5de15eeb..c38202e13 100644 --- a/deploy-chroot/setup.yaml.in +++ b/deploy-chroot/setup.yaml.in @@ -10,9 +10,11 @@ cluster: vlan: 612 addresses: - cidr-nodes: 10.{x}.0.0/24 + cidr-nodes: 10.{x}.0.0/23 cidr-pods: 172.18.0.0/16 cidr-services: 172.28.0.0/16 + external-ranges: + - 10.{x}.1.0/24 service-api: 172.28.0.1 service-dns: 172.28.0.2 diff --git a/docs/cluster-configuration.md b/docs/cluster-configuration.md index 976f2dd12..8575c5867 100644 --- a/docs/cluster-configuration.md +++ b/docs/cluster-configuration.md @@ -97,6 +97,8 @@ Sample section: cidr-nodes: 18.4.60.0/23 cidr-pods: 172.18.0.0/16 cidr-services: 172.28.0.0/16 + external-ranges: + - 18.4.62.0/16 service-api: 172.28.0.1 service-dns: 172.28.0.2 @@ -121,6 +123,10 @@ Descriptions: This should be a subnet like `cidr-pods`, but can be smaller, because you will generally have many fewer services than pods. These are allocated to services across the cluster as needed. + * `external-ranges`: the list of publicly-addressible subnets that IP addresses for LoadBalancer services should be allocated out of. + + These may overlap with `cidr-nodes`. + * `service-api`: the service IP address for the Kubernetes apiserver within the cluster. This must always be within the service subnet, and is usually the first address, but can be anything within the range diff --git a/platform/WORKSPACE b/platform/WORKSPACE index 74ec400b5..64de16b4b 100644 --- a/platform/WORKSPACE +++ b/platform/WORKSPACE @@ -88,6 +88,10 @@ load("//kube-state-metrics:deps.bzl", "kube_state_metrics_dependencies") kube_state_metrics_dependencies() +load("//metallb:deps.bzl", "metallb_dependencies") + +metallb_dependencies() + load("//oci-tools:deps.bzl", "oci_tools_dependencies") oci_tools_dependencies() diff --git a/platform/metallb/BUILD.bazel b/platform/metallb/BUILD.bazel new file mode 100644 index 000000000..b281f6962 --- /dev/null +++ b/platform/metallb/BUILD.bazel @@ -0,0 +1,47 @@ +load("//bazel:package.bzl", "homeworld_oci") +load("//bazel:substitute.bzl", "substitute") +load("//python:resources.bzl", "py_resources") + +# TODO: ldflags: +# -X go.universe.tf/metallb/internal/version.gitCommit={commit} +# -X go.universe.tf/metallb/internal/version.gitBranch={branch} + +homeworld_oci( + name = "controller", + bin = { + "@tf_universe_go_metallb//controller": "/usr/bin/controller", + }, + exec = ["/usr/bin/controller"], + visibility = ["//visibility:public"], + deps = [ + "//debian:debian-micro.tgz", + ], +) + +homeworld_oci( + name = "speaker", + bin = { + "@tf_universe_go_metallb//speaker": "/usr/bin/speaker", + }, + exec = ["/usr/bin/speaker"], + visibility = ["//visibility:public"], + deps = [ + "//debian:debian-micro.tgz", + ], +) + +substitute( + name = "kubernetes.yaml", + kfs = { + "controller_digest": ":controller.ocidigest", + "speaker_digest": ":speaker.ocidigest", + }, + template = ":kubernetes.yaml.in", + visibility = ["//visibility:public"], +) + +py_resources( + name = "kubelib", + data = [":kubernetes.yaml"], + visibility = ["//visibility:public"], +) diff --git a/platform/metallb/deps.bzl b/platform/metallb/deps.bzl new file mode 100644 index 000000000..04622632a --- /dev/null +++ b/platform/metallb/deps.bzl @@ -0,0 +1,68 @@ +load("@bazel_gazelle//:deps.bzl", "go_repository") + +def metallb_dependencies(): + go_repository( + name = "tf_universe_go_metallb", + commit = "15d9ed5cce53b457a955f894aa58e7f4856f73d5", # v0.8.3 + importpath = "go.universe.tf/metallb", + ) + + go_repository( + name = "com_github_go_kit_kit", + commit = "150a65a7ec6156b4b640c1fd55f26fd3d475d656", # v0.9.0 + importpath = "github.com/go-kit/kit", + ) + + go_repository( + name = "com_github_go_logfmt_logfmt", + commit = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5", # v0.3.0 + importpath = "github.com/go-logfmt/logfmt", + ) + + go_repository( + name = "com_github_golang_groupcache", + commit = "02826c3e79038b59d737d3b1c0a1d937f71a4433", + importpath = "github.com/golang/groupcache", + ) + + go_repository( + name = "com_github_hashicorp_golang_lru", + commit = "20f1fb78b0740ba8c3cb143a61e86ba5c8669768", # v0.5.0 + importpath = "github.com/hashicorp/golang-lru", + ) + + go_repository( + name = "com_github_mdlayher_arp", + commit = "98a83c8a27177c5179d02d41ad50b0cce8e59338", + importpath = "github.com/mdlayher/arp", + ) + + go_repository( + name = "com_github_mdlayher_ethernet", + commit = "0394541c37b7f86a10e0b49492f6d4f605c34163", + importpath = "github.com/mdlayher/ethernet", + ) + + go_repository( + name = "com_github_mdlayher_ndp", + commit = "012988d57f9ae7e329f16ec2a86b37c771cb57e8", + importpath = "github.com/mdlayher/ndp", + ) + + go_repository( + name = "com_github_mdlayher_raw", + commit = "fef19f00fc18511f735e13972bc53266d5a53f8c", + importpath = "github.com/mdlayher/raw", + ) + + go_repository( + name = "com_github_mikioh_ipaddr", + commit = "d465c8ab672111787b24b8f03326449059a4aa33", + importpath = "github.com/mikioh/ipaddr", + ) + + go_repository( + name = "com_gitlab_golang_commonmark_puny", + commit = "2cd490539afe7c6fc0eda6c59ef88fa93a00ea0d", + importpath = "gitlab.com/golang-commonmark/puny", + ) diff --git a/platform/metallb/kubernetes.yaml.in b/platform/metallb/kubernetes.yaml.in new file mode 100644 index 000000000..716988743 --- /dev/null +++ b/platform/metallb/kubernetes.yaml.in @@ -0,0 +1,401 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app: metallb + name: metallb-system +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +spec: + allowPrivilegeEscalation: false + allowedCapabilities: [] + allowedHostPaths: [] + defaultAddCapabilities: [] + defaultAllowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + hostIPC: false + hostNetwork: false + hostPID: false + privileged: false + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL + runAsUser: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - secret + - emptyDir +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + allowedHostPaths: [] + defaultAddCapabilities: [] + defaultAllowPrivilegeEscalation: false + fsGroup: + rule: RunAsAny + hostIPC: false + hostNetwork: true + hostPID: false + hostPorts: + - max: 7472 + min: 7472 + privileged: true + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - secret + - emptyDir +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:controller +rules: +- apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - update +- apiGroups: + - '' + resources: + - services/status + verbs: + - update +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - extensions + resourceNames: + - controller + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:speaker +rules: +- apiGroups: + - '' + resources: + - services + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - extensions + resourceNames: + - speaker + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:speaker +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: config-watcher +subjects: +- kind: ServiceAccount + name: controller +- kind: ServiceAccount + name: speaker +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: metallb + component: speaker + name: speaker + namespace: metallb-system +spec: + selector: + matchLabels: + app: metallb + component: speaker + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: speaker + spec: + containers: + - args: + - /usr/bin/speaker + - --port=7472 + - --config=config + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + image: homeworld.private/metallb-speaker@{speaker_digest} + imagePullPolicy: Always + name: speaker + ports: + - containerPort: 7472 + name: monitoring + resources: + limits: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + drop: + - ALL + readOnlyRootFilesystem: true + hostNetwork: true + nodeSelector: + beta.kubernetes.io/os: linux + serviceAccountName: speaker + terminationGracePeriodSeconds: 0 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: metallb + component: controller + name: controller + namespace: metallb-system +spec: + revisionHistoryLimit: 3 + selector: + matchLabels: + app: metallb + component: controller + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: controller + spec: + containers: + - args: + - /usr/bin/controller + - --port=7472 + - --config=config + image: homeworld.private/metallb-controller@{controller_digest} + imagePullPolicy: Always + name: controller + ports: + - containerPort: 7472 + name: monitoring + resources: + limits: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + nodeSelector: + beta.kubernetes.io/os: linux + securityContext: + # FIXME(#454): uncomment these lines + # runAsNonRoot: true + # runAsUser: 65534 + serviceAccountName: controller + terminationGracePeriodSeconds: 0 +# Above this line is adapted from metallb's manifest +--- +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: {{EXTERNAL_RANGES}} +--- +apiVersion: v1 +kind: Service +metadata: + name: metallb-controller + namespace: metallb-system + labels: + app: metallb +spec: + selector: + app: metallb + component: controller + ports: + - name: metrics + port: 7472 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: metallb-speaker + namespace: metallb-system + labels: + app: metallb +spec: + selector: + app: metallb + component: speaker + ports: + - name: metrics + port: 7472 + protocol: TCP diff --git a/platform/spire/BUILD.bazel b/platform/spire/BUILD.bazel index 5b733cf41..365a1c3c0 100644 --- a/platform/spire/BUILD.bazel +++ b/platform/spire/BUILD.bazel @@ -13,6 +13,7 @@ py_binary( "//dnsmasq:kubelib", "//flannel:kubelib", "//flannel-monitor:kubelib", + "//metallb:kubelib", "//spire/resources:resourcelib", "//upload:uploadlib", "//user-grant:kubelib", diff --git a/platform/spire/resources/prometheus.yaml b/platform/spire/resources/prometheus.yaml index 5de9af554..ad0ebc997 100644 --- a/platform/spire/resources/prometheus.yaml +++ b/platform/spire/resources/prometheus.yaml @@ -135,3 +135,23 @@ scrape_configs: regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + - job_name: 'metallb-controller' + scheme: https + tls_config: + ca_file: /etc/homeworld/authorities/kubernetes.pem + cert_file: /etc/homeworld/keys/kubernetes-supervisor.pem + key_file: /etc/homeworld/keys/kubernetes-supervisor.key + static_configs: + - targets: ['{{APISERVER}}:443'] + metrics_path: '/api/v1/namespaces/metallb-system/services/metallb-controller:metrics/proxy/metrics' + + - job_name: 'metallb-speaker' + scheme: https + tls_config: + ca_file: /etc/homeworld/authorities/kubernetes.pem + cert_file: /etc/homeworld/keys/kubernetes-supervisor.pem + key_file: /etc/homeworld/keys/kubernetes-supervisor.key + static_configs: + - targets: ['{{APISERVER}}:443'] + metrics_path: '/api/v1/namespaces/metallb-system/services/metallb-speaker:metrics/proxy/metrics' diff --git a/platform/spire/resources/setup-schema.yaml b/platform/spire/resources/setup-schema.yaml index 8610708ea..1edbfc448 100644 --- a/platform/spire/resources/setup-schema.yaml +++ b/platform/spire/resources/setup-schema.yaml @@ -34,11 +34,15 @@ properties: type: string cidr-services: type: string + external-ranges: + type: array + items: + type: string service-api: type: string service-dns: type: string - required: ["cidr-nodes", "cidr-pods", "cidr-services", "service-api", "service-dns"] + required: ["cidr-nodes", "cidr-pods", "cidr-services", "external-ranges", "service-api", "service-dns"] additionalProperties: false dns-upstreams: type: array diff --git a/platform/spire/resources/setup.yaml b/platform/spire/resources/setup.yaml index 7546d92d7..b6c363974 100644 --- a/platform/spire/resources/setup.yaml +++ b/platform/spire/resources/setup.yaml @@ -13,6 +13,7 @@ addresses: cidr-nodes: 18.4.60.0/23 cidr-pods: 172.18.0.0/16 cidr-services: 172.28.0.0/16 + external_ranges: [] service-api: 172.28.0.1 service-dns: 172.28.0.2 diff --git a/platform/spire/src/configuration.py b/platform/spire/src/configuration.py index 889eb9dab..408e29453 100644 --- a/platform/spire/src/configuration.py +++ b/platform/spire/src/configuration.py @@ -67,6 +67,7 @@ def __init__(self, kv: dict): self.cidr_nodes = IPv4Network(kv["addresses"]["cidr-nodes"]) self.cidr_pods = IPv4Network(kv["addresses"]["cidr-pods"]) self.cidr_services = IPv4Network(kv["addresses"]["cidr-services"]) + self.external_ranges = [IPv4Network(x) for x in kv["addresses"]["external-ranges"]] self.service_api = IPv4Address(kv["addresses"]["service-api"]) self.service_dns = IPv4Address(kv["addresses"]["service-dns"]) diff --git a/platform/spire/src/deploy.py b/platform/spire/src/deploy.py index f55bc4c59..b83cdee37 100644 --- a/platform/spire/src/deploy.py +++ b/platform/spire/src/deploy.py @@ -60,6 +60,15 @@ def launch_dns_monitor(export: bool=False): launch_spec("//dns-monitor:kubernetes.yaml", export=export) +@command.wrap +def launch_metallb(export: bool=False): + "deploy the specifications to run metallb" + config = configuration.get_config() + launch_spec("//metallb:kubernetes.yaml", { + "EXTERNAL_RANGES": [str(x) for x in config.external_ranges], + }, export=export) + + @command.wrap def launch_user_grant(export: bool=False): "deploy the specifications to run the user grant website" @@ -94,5 +103,6 @@ def launch_user_grant(export: bool=False): "flannel-monitor": launch_flannel_monitor, "dns-addon": launch_dns_addon, "dns-monitor": launch_dns_monitor, + "metallb": launch_metallb, "user-grant": launch_user_grant, }) diff --git a/platform/spire/src/seq.py b/platform/spire/src/seq.py index 3d1f15dee..1b46e5f52 100644 --- a/platform/spire/src/seq.py +++ b/platform/spire/src/seq.py @@ -50,6 +50,7 @@ def sequence_supervisor(ops: command.Operations, skip_verify_keygateway: bool=Fa ops.add_command(deploy.launch_dns_addon) ops.add_command(deploy.launch_flannel_monitor) ops.add_command(deploy.launch_dns_monitor) + ops.add_command(deploy.launch_metallb) if config.user_grant_domain != '': ops.add_command(deploy.launch_user_grant) @@ -124,6 +125,8 @@ def sequence_cluster(ops: command.Operations) -> None: else: ops.add_operation("verify that user-grant is working properly", iterative_verifier(verify.check_user_grant, 120.0)) + ops.add_command(iterative_verifier(verify.check_metallb, 120.0)) + main_command = command.SeqMux("commands about running large sequences of cluster bring-up automatically", { "keysystem": sequence_keysystem, diff --git a/platform/spire/src/verify.py b/platform/spire/src/verify.py index 905334821..f9ea8d621 100644 --- a/platform/spire/src/verify.py +++ b/platform/spire/src/verify.py @@ -353,6 +353,13 @@ def check_user_grant(): print("autogenerated rolebinding for user", repr(authority.UPSTREAM_USER_NAME), "passed basic check!") +@command.wrap +def check_metallb(): + "verify that metallb is running" + expect_prometheus_query_exact('sum(up{job="metallb-controller"})', 1, "metallb controller is online") + expect_prometheus_query_exact('sum(up{job="metallb-speaker"})', 1, "metallb speaker is online") + print("metallb is running") + main_command = command.Mux("commands about verifying the state of a cluster", { "keystatics": check_keystatics, @@ -369,4 +376,5 @@ def check_user_grant(): "flannel": check_flannel, "dns-addon": check_dns, "user-grant": check_user_grant, + "metallb": check_metallb, }) diff --git a/platform/upload/BUILD.bazel b/platform/upload/BUILD.bazel index 88bd3f81b..c569f0225 100644 --- a/platform/upload/BUILD.bazel +++ b/platform/upload/BUILD.bazel @@ -11,6 +11,8 @@ ocis = { "flannel-monitor": "//flannel-monitor:oci", "kube-dns-main": "//kube-dns:kube-dns-main", "kube-dns-sidecar": "//kube-dns:kube-dns-sidecar", + "metallb-controller": "//metallb:controller", + "metallb-speaker": "//metallb:speaker", "pause": "//cri-o/pause:oci", "user-grant": "//user-grant:oci", }