From edaadeab6480fa39d1b7ddb758add6f08f3735ad Mon Sep 17 00:00:00 2001 From: Satya Ranjan Pradhan Date: Tue, 15 Jan 2019 19:48:45 +0530 Subject: [PATCH] Version 3.0 release --- releases/v3.0/CNIGenieFeatureSet.md | 32 ++++++ releases/v3.0/GettingStarted.md | 80 ++++++++++++++ releases/v3.0/README.md | 76 +++++++++++++ releases/v3.0/developer-guide.md | 38 +++++++ releases/v3.0/genie.yaml | 163 ++++++++++++++++++++++++++++ 5 files changed, 389 insertions(+) create mode 100644 releases/v3.0/CNIGenieFeatureSet.md create mode 100644 releases/v3.0/GettingStarted.md create mode 100644 releases/v3.0/README.md create mode 100644 releases/v3.0/developer-guide.md create mode 100644 releases/v3.0/genie.yaml diff --git a/releases/v3.0/CNIGenieFeatureSet.md b/releases/v3.0/CNIGenieFeatureSet.md new file mode 100644 index 00000000..8b6c4529 --- /dev/null +++ b/releases/v3.0/CNIGenieFeatureSet.md @@ -0,0 +1,32 @@ +# Features covered in each CNI-Genie version: + +## Existing features + +**Feature 1: CNI-Genie "Multiple CNI Plugins"** +* Interface Connector to 3rd party CNI-Plugins. The user can [manually select one of the multiple CNI plugins](multiple-cni-plugins/README.md) + +**Feature 2: CNI-Genie "Multiple IP Addresses"** +* Injects multiple IPs to a single container. The container is reachable using any of the [multiple IP Addresses](multiple-ips/README.md) + +**Feature 3: CNI-Genie "Network Attachment Definition"** +* [Network Attachment Definition](network-attachment-definitions/README.md) feature incorporates Kubernetes Network Custom Resource Definition De-facto Standard in CNI-Genie + +**Feature 4: CNI-Genie "Smart CNI Plugin Selection"** +* Intelligence in selecting the CNI plugin. CNI-Genie [watches the KPI of interest and selects](smart-cni-genie/README.md) the CNI plugin, accordingly + +**Feature 5: CNI-Genie "Default Plugin Selection"** +* Support to set default plugin of user choice to be used for all the pods being created + +**Feature 6: CNI-Genie "Network Isolation"** +* Dedicated 'physical' network for a tenant +* Isolated 'logical' networks for different tenants on a shared 'physical'network + +## Future features + +**Feature 7: CNI-Genie "Network Policy Engine"** +* [CNI-Genie network policy engine](network-policy/README.md) allows for network level ACLs + +**Feature 8: CNI-Genie "Real-time Network Switching"** +* Price minimization: dynamically switching workload to a cheaper network as network prices change +* Maximizing network utilization: dynamically switching workload to the less congested network at a threshold + diff --git a/releases/v3.0/GettingStarted.md b/releases/v3.0/GettingStarted.md new file mode 100644 index 00000000..0fc862b2 --- /dev/null +++ b/releases/v3.0/GettingStarted.md @@ -0,0 +1,80 @@ +# Getting started + +### Prerequisite + +* Linux box with + * We tested on Ubuntu 14.04 & 16.04 +* Docker installed +* Kubernetes cluster running with CNI enabled + * One easy way to bring up a cluster is to use [kubeadm](https://kubernetes.io/docs/getting-started-guides/kubeadm/): + * We tested on Kubernetes 1.5, 1.6, 1.7, 1.8 + + Till 1.7 version: + ``` + $ kubeadm init --use-kubernetes-version=v1.7.0 --pod-network-cidr=10.244.0.0/16 + ``` + + Version 1.8 onwards: + ``` + $ kubeadm init --pod-network-cidr=10.244.0.0/16 + ``` + + Next steps: + ``` + $ mkdir -p $HOME/.kube + $ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config + $ sudo chown $(id -u):$(id -g) $HOME/.kube/config + ``` + * To schedule pods on the master, e.g. for a single-machine Kubernetes cluster, + + Till 1.7 version, run: + ``` + $ kubectl taint nodes --all dedicated- + ``` + + Version 1.8 onwards, run: + ``` + $ kubectl taint nodes --all node-role.kubernetes.io/master- + ``` + + +* One (or more) CNI plugin(s) installed, e.g., Calico, Weave, Flannel + * Use this [link](https://docs.projectcalico.org/v3.2/getting-started/kubernetes) to install Calico + * Use this [link](https://www.weave.works/docs/net/latest/kube-addon/) to install Weave + * Use this [link](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml) to install Flannel + +### Installing genie + +We install genie as a Docker Container on every node + +Till Kubernetes 1.7 version: +``` +$ kubectl apply -f https://raw.githubusercontent.com/Huawei-PaaS/CNI-Genie/master/conf/1.5/genie.yaml +``` + +Kubernetes 1.8 version onwards: +``` +$ kubectl apply -f https://raw.githubusercontent.com/Huawei-PaaS/CNI-Genie/master/releases/v3.0/genie.yaml +``` + +### Building, Testing, Making changes to source code + +Refer to our [Developer's Guide](developer-guide.md) section. + +### Genie Logs + +For now Genie logs are stored in /var/log/syslog +To see the logs: +``` +$ cat /dev/null > /var/log/syslog + +$ tail -f /var/log/syslog | grep 'CNI' +``` + +### Troubleshooting + +* Note: one a single node cluster, after your Kubernetes master is initialized successfully, make sure you are able to schedule pods on the master by running: +``` +$ kubectl taint nodes --all node-role.kubernetes.io/master- +``` +* Note: most plugins use differenet installation files for Kuberenetes 1.5, 1.6, 1.7 & 1.8. Make sure you use the right one! diff --git a/releases/v3.0/README.md b/releases/v3.0/README.md new file mode 100644 index 00000000..57121467 --- /dev/null +++ b/releases/v3.0/README.md @@ -0,0 +1,76 @@ +# CNI-Genie + +CNI-Genie enables container orchestrators ([Kubernetes](https://github.com/kubernetes/kubernetes), [Mesos](https://mesosphere.com/)) to seamlessly connect to the choice of CNI plugins installed on a host, including +1. ['reference' CNI plugins](https://github.com/containernetworking/plugins), e.g., bridge, macvlan, ipvlan, loopback +2. '3rd-party' CNI plugins, e.g., ([Calico](https://github.com/projectcalico/calico), [Romana](https://github.com/romana/romana), [Weave-net](https://github.com/weaveworks/weave)) +3. 'specialized' CNI plugins, e.g., [SR-IOV](https://github.com/hustcat/sriov-cni), DPDK (work-in-progress) +4. any generic CNI plugin of choice installed on the host + +Without CNI-Genie, the orchestrator is bound to only a single CNI plugin. E.g., for the case of Kubernetes, without CNI-Genie, kubelet is bound to only a single CNI plugin passed to kubelet on start. CNI-Genie allows for the co-existance of multiple CNI plugins in runtime. + +[![Build Status](https://travis-ci.org/Huawei-PaaS/CNI-Genie.svg)](https://travis-ci.org/Huawei-PaaS/CNI-Genie) +[![Go Report Card](https://goreportcard.com/badge/github.com/Huawei-PaaS/CNI-Genie)](https://goreportcard.com/report/github.com/Huawei-PaaS/CNI-Genie) + +Please feel free to post your feedback, questions on CNI-Genie [Slack channel](https://cni-genie.slack.com/) + +## Demo +Here is a 6 minute demo video that demonstrates 3 scenarios +1. Assign an IP address to a pod from a particular network solution, e.g., 'Weave-net' +2. Assign multi-IP addresses to a pod from multiple network solutions, e.g., 1st IP address from 'Weave-net', 2nd IP address from 'Canal' +3. Assign an IP address to a pod from the "less congested" network solution, e.g., from 'Canal' that is less congested + +[![asciicast](https://asciinema.org/a/118191.png)](https://asciinema.org/a/118191) + +# Contributing +[Contributing](../../CONTRIBUTING.md) + +[Code of Conduct](../../CODE_OF_CONDUCT.md) + +# Why we created CNI-Genie? + +CNI Genie is an add-on to [Kuberenets](https://github.com/kubernetes/kubernetes) open-source project and is designed to provide the following features: + +1. [wide range of network offerings, CNI plugins](../../docs/multiple-cni-plugins/README.md), available to the users in runtime. This figure shows Kubernetes CNI Plugin landscape before and after CNI-Genie + ![image](../../docs/multiple-cni-plugins/what-cni-genie.png) + - User-story: based on "performance" requirements, "application" requirements, “workload placement” requirements, the user could be interested to use different CNI plugins for different application groups + - Different CNI plugins are different in terms of need for port-mapping, NAT, tunneling, interrupting host ports/interfaces + +[Watch multiple CNI plugins demo](https://github.com/Huawei-PaaS/CNI-Genie/blob/master/docs/multiple-cni-plugins/README.md#demo) + + +2. [Multiple NICs per container & per pod](../../docs/multiple-ips/README.md). The user can select multiple NICs to be added to a container upon creating them. Each NIC can get an IP address from an existing CNI plugin of choice. This makes the container reachable across multiple networks. Some use-cases from [SIG-Network](https://github.com/kubernetes/community/wiki/SIG-Network) are depicted in the figure below + ![image](../../docs/multiple-ips/multi-interface.PNG) + +[Watch multi-NICs per 'container' demo](https://github.com/Huawei-PaaS/CNI-Genie/blob/master/docs/multiple-ips/README.md#demo) + +[Watch multi-NICs per 'pod' demo](https://github.com/Huawei-PaaS/CNI-Genie/blob/master/docs/multiple-ips/README.md#feature-2-extension-cni-genie-multiple-ip-addresses-per-pod) (IP addresses assigned not only to the container, but also to the Pod) + +3. [Network Attachment Definition](../../docs/network-attachment-definitions/README.md). CNI-Genie supports [NPWG Multi-Network Specification v1](https://github.com/K8sNetworkPlumbingWG/multi-net-spec) style network attachment to pods, where pods can be assigned IP according to network-attachment-definition CRD objects created by user. + +4. The user can leave the CNI plugin selection to CNI-Genie. CNI-Genie watches the Key Performance Indicator (KPI) that is of interest to the user and [selects the CNI plugin](../../docs/smart-cni-genie/README.md), accordingly. + - CNI Genie watches KPI(s) of interest for existing CNI plugins, e.g., occupancy rate, number of subnets, latency, bandwidth + +[Watch Smart CNI Plugin Selection demo](https://github.com/Huawei-PaaS/CNI-Genie/blob/master/docs/smart-cni-genie/README.md#demo) + + +5. [Default plugin support](../../docs/default-plugin/README.md). Another useful feature from genie. Using this, we can ensure to get ip address(es) for a pod by selecting default set of plugins + + +6. Network isolation, i.e., + - Dedicated 'physical' network for a tenant + - Isolated 'logical' networks for different tenants on a shared 'physical'network + + Usecase : [Obtaining Pod Ip address from customized subnet](../../docs/network-isolation/README.md) + +7. [CNI-Genie network policy engine](../../docs/network-policy/README.md) for network level ACLs + +8. Real-time switching between different (physical or logical) networks for a given workload. This allows for + - Price minimization: dynamically switching workload to a cheaper network as network prices change + - Maximizing network utilization: dynamically switching workload to the less congested network at a threshold + + ![image](../../docs/network-switching.PNG) + +Note: CNI-Genie itself is NOT a routing solution! It makes a call to CNI plugins that provide routing service + +### More docs here [Getting started](GettingStarted.md), [CNI-Genie Feature Set](CNIGenieFeatureSet.md) + diff --git a/releases/v3.0/developer-guide.md b/releases/v3.0/developer-guide.md new file mode 100644 index 00000000..2f86678d --- /dev/null +++ b/releases/v3.0/developer-guide.md @@ -0,0 +1,38 @@ +## Developer's Guide + +### Build process + +After making any modification to source files, below steps can be followed to build and use the new binary. + +Note that you should install genie first before making changes to the source. This ensures genie conf file is generated successfully. + +Please make sure to run the below commands with root privilege. + +#### *Building and Using CNI-Genie plugin:* + +Build genie binary by running: +``` +make plugin +``` +Place "genie" binary from dest/ into /opt/cni/bin/ directory. +``` +cp dist/genie /opt/cni/bin/genie +``` +### Test process + +#### prerequisites + +A running kubernetes cluster is required to run the tests. + +#### Running the tests + +To run ginkgo tests for CNI-Genie run the following command: + +If Kubernetes cluster is 1.7+ +``` +make test testKubeVersion=1.7 testKubeConfig=/etc/kubernetes/admin.conf +``` + +If Kubernetes cluster is 1.5.x +``` +make test testKubeVersion=1.5 diff --git a/releases/v3.0/genie.yaml b/releases/v3.0/genie.yaml new file mode 100644 index 00000000..a35416c5 --- /dev/null +++ b/releases/v3.0/genie.yaml @@ -0,0 +1,163 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: genie +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - update + - patch + - apiGroups: + - "alpha.network.k8s.io" + resources: + - logicalnetworks + verbs: + - get + - update + - patch + - apiGroups: + - "alpha.network.k8s.io" + resources: + - physicalnetworks + verbs: + - get + - update + - patch + - apiGroups: + - "k8s.cni.cncf.io" + resources: + - network-attachment-definitions + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - update + - create + - delete + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: genie +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: genie +subjects: +- kind: ServiceAccount + name: genie + namespace: kube-system +- kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: genie + namespace: kube-system + +--- +# This ConfigMap can be used to configure a self-hosted CNI-Genie installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: genie-config + namespace: kube-system +data: + # The CNI network configuration to install on each node. + cni_genie_network_config: |- + { + "name": "k8s-pod-network", + "type": "genie", + "log_level": "info", + "datastore_type": "kubernetes", + "hostname": "__KUBERNETES_NODE_NAME__", + "policy": { + "type": "k8s", + "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" + }, + "kubernetes": { + "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", + "kubeconfig": "/etc/cni/net.d/genie-kubeconfig" + }, + "romana_root": "http://__ROMANA_SERVICE_HOST__:__ROMANA_SERVICE_PORT__", + "segment_label_name": "romanaSegment" + } +--- +# Install CNI-Genie plugin on each slave node. +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: genie-plugin + namespace: kube-system + labels: + k8s-app: genie +spec: + selector: + matchLabels: + k8s-app: genie + template: + metadata: + labels: + k8s-app: genie + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + hostPID: true + serviceAccountName: genie + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + containers: + # Create a container with install.sh that + # Installs required 00-genie.conf and genie binary + # on slave node. + - name: install-cni + image: cnigenie/genie-plugin:v3.0 + imagePullPolicy: Always + command: ["/launch.sh"] + env: + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: genie-config + key: cni_genie_network_config + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + volumes: + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d