apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    app: node-metrics
  name: node-metrics
  namespace: crane-system
spec:
  selector:
    matchLabels:
      app: node-metrics
  template:
    metadata:
      labels:
        app: node-metrics
    spec:
      containers:
      - image: dongjiang1989/node-metrics:latest
        name: node-metrics
        args:
        - --web.listen-address=0.0.0.0:19101
        resources:
          limits:
            cpu: 102m
            memory: 180Mi
          requests:
            cpu: 102m
            memory: 180Mi
      hostNetwork: true
      hostPID: true
      tolerations:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master

...
# HELP node_cpu_usage_active cpu usage active.
# TYPE node_cpu_usage_active gauge
node_cpu_usage_active 6.801955214695443
# HELP node_cpu_usage_avg_5m cpu usage avg 5m.
# TYPE node_cpu_usage_avg_5m gauge
node_cpu_usage_avg_5m 6.8018810008297335
# HELP node_cpu_usage_max_avg_1d cpu usage max avg 1d.
# TYPE node_cpu_usage_max_avg_1d gauge
node_cpu_usage_max_avg_1d 6.801955214695443
# HELP node_cpu_usage_max_avg_1h cpu usage max avg 1h.
# TYPE node_cpu_usage_max_avg_1h gauge
node_cpu_usage_max_avg_1h 6.801955214695443
# HELP node_mem_usage_active mem usage active.
# TYPE node_mem_usage_active gauge
node_mem_usage_active 44.272822236553765
# HELP node_mem_usage_avg_5m mem usage avg 5m.
# TYPE node_mem_usage_avg_5m gauge
node_mem_usage_avg_5m 43.68676937682602
# HELP node_mem_usage_max_avg_1d mem usage max avg 1d.
# TYPE node_mem_usage_max_avg_1d gauge
node_mem_usage_max_avg_1d 44.447325557125225
# HELP node_mem_usage_max_avg_1h mem usage max avg 1h.
# TYPE node_mem_usage_max_avg_1h gauge
node_mem_usage_max_avg_1h 44.447325557125225
...

{
  "cpu_usage_active": 6.801955214695443,
  "cpu_usage_avg_5m": 6.8018810008297335,
  "cpu_usage_max_avg_1d": 6.801955214695443,
  "cpu_usage_max_avg_1h": 6.801955214695443,
  "mem_usage_active": 44.272822236553765,
  "mem_usage_avg_5m": 43.68676937682602,
  "mem_usage_max_avg_1d": 44.447325557125225,
  "mem_usage_max_avg_1h": 44.447325557125225
}

type Data struct {
    Name     string
    BirthDay time.Time
    Phone    string
    Siblings int
    Spouse   bool
    Money    float64
    ExInfo   interface{}
}

cd pingmesh-heatmap-panel/
yarn install
grunt

cd /var/lib/grafana/plugins
git clone https://github.com/kubeservice-stack/pingmesh-heatmap-panel.git
sudo service grafana-server restart

$ chmod 777 /var/lib/grafana/plugins/pingmesh-heatmap-panel/

metrics:
  counters:
    - name: timer_starts_total
      help: Timers fired in the kernel
      labels:
        - name: function
          size: 8
          decoders:
            - name: ksym

#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include "maps.bpf.h"

struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 1024);
    __type(key, u64);
    __type(value, u64);
} timer_starts_total SEC(".maps");

SEC("tracepoint/timer/timer_start") // 使用tracepoint跟踪timer_start，其中，跟踪点是timer:timer_start
int do_count(struct trace_event_raw_timer_start* ctx)
{
    u64 function = (u64) ctx->function;

    increment_map(&timer_starts_total, &function, 1);

    return 0;
}

char LICENSE[] SEC("license") = "GPL"; // 内核要求必须是GPL license

metrics:
  counters:
    - name: timer_starts_total
      help: Timers fired in the kernel
      labels:
        - name: function
          size: 8
          decoders:
            - name: ksym

SEC("tracepoint/timer/timer_start") // 使用tracepoint跟踪timer_start，其中，跟踪点是timer:timer_start

int do_count(struct trace_event_raw_timer_start* ctx)
{
    u64 function = (u64) ctx->function;

    increment_map(&timer_starts_total, &function, 1);

    return 0;
}

dongjiang@MacBook Pro:~ $  cosign sign dongjiang1989/node-metrics@sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282
Generating ephemeral keys...
Retrieving signed certificate...

	The sigstore service, hosted by sigstore a Series of LF Projects, LLC, is provided pursuant to the Hosted Project Tools Terms of Use, available at https://lfprojects.org/policies/hosted-project-tools-terms-of-use/.
	Note that if your submission includes personal data associated with this signed artifact, it will be part of an immutable record.
	This may include the email address associated with the account with which you authenticate your contractual Agreement.
	This information will be used for signing this artifact and will be stored in public transparency logs and cannot be removed later, and is subject to the Immutable Record notice at https://lfprojects.org/policies/hosted-project-tools-immutable-records/.

By typing 'y', you attest that (1) you are not submitting the personal data of any other person; and (2) you understand and agree to the statement and the Agreement terms at the URLs listed above.
Are you sure you would like to continue? [y/N] y
error opening browser: exec: "xdg-open": executable file not found in $PATH
Go to the following link in a browser:

	 https://oauth2.sigstore.dev/auth/auth?access_type=online&client_id=sigstore&code_challenge=L_clEcpY7BXtgaJ0W5qZJq74ovXNnznXpwYluzCUPEQ&code_challenge_method=S256&nonce=2TSKHjXGVJMxbj4TgFNe6wtXXKg&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&response_type=code&scope=openid+email&state=2TSKHg5tyauZ3PaaVJHyNfRAf1D
Enter verification code: yql4si3wqdvkzrm25ohncuk45

Successfully verified SCT...
tlog entry created with index: 29886419
Pushing signature to: index.docker.io/dongjiang1989/node-metrics

dongjiang@MacBook Pro:~ $ cosign verify --certificate-identity dongjiang1989@126.com --certificate-oidc-issuer https://github.com/login/oauth dongjiang1989/node-metrics@sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282

Verification for index.docker.io/dongjiang1989/node-metrics@sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282 --
The following checks were performed on each of these signatures:
  - The cosign claims were validated
  - Existence of the claims in the transparency log was verified offline
  - The code-signing certificate was verified using trusted certificate authority certificates

[{"critical":{"identity":{"docker-reference":"index.docker.io/dongjiang1989/node-metrics"},"image":{"docker-manifest-digest":"sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282"},"type":"cosign container image signature"},"optional":{"1.3.6.1.4.1.57264.1.1":"https://github.com/login/oauth","Bundle":{"SignedEntryTimestamp":"xxxxxxxxx","Payload":{"body":"xxxxxxxxxxxx=","integratedTime":1691030975,"logIndex":29886419,"logID":"c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d"}},"Issuer":"https://github.com/login/oauth","Subject":"dongjiang1989@126.com"}}]

dongjiang@MacBook Pro:~ $ cosign generate-key-pair
Enter password for private key: 
Enter password for private key again: 
Private key written to cosign.key
Public key written to cosign.pub
dongjiang@MacBook Pro:~ $ ls
aaa  cosign.key  cosign.pub  go  src  tools

dongjiang@MacBook Pro:~ $ cosign sign --key cosign.key dongjiang1989/node-metrics:sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282
Enter password for private key: 
Error: signing [dongjiang1989/node-metrics:sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282]: parsing reference: could not parse reference: dongjiang1989/node-metrics:sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282
main.go:74: error during command execution: signing [dongjiang1989/node-metrics:sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282]: parsing reference: could not parse reference: dongjiang1989/node-metrics:sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282
[root@VM-0-5-centos ~]# cosign sign --key cosign.key dongjiang1989/node-metrics@sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282
Enter password for private key: 

	The sigstore service, hosted by sigstore a Series of LF Projects, LLC, is provided pursuant to the Hosted Project Tools Terms of Use, available at https://lfprojects.org/policies/hosted-project-tools-terms-of-use/.
	Note that if your submission includes personal data associated with this signed artifact, it will be part of an immutable record.
	This may include the email address associated with the account with which you authenticate your contractual Agreement.
	This information will be used for signing this artifact and will be stored in public transparency logs and cannot be removed later, and is subject to the Immutable Record notice at https://lfprojects.org/policies/hosted-project-tools-immutable-records/.

By typing 'y', you attest that (1) you are not submitting the personal data of any other person; and (2) you understand and agree to the statement and the Agreement terms at the URLs listed above.
Are you sure you would like to continue? [y/N] y
tlog entry created with index: 29887689
Pushing signature to: index.docker.io/dongjiang1989/node-metrics

[root@VM-0-5-centos ~]# cosign verify --key cosign.pub dongjiang1989/node-metrics@sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282

Verification for index.docker.io/dongjiang1989/node-metrics@sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282 --
The following checks were performed on each of these signatures:
  - The cosign claims were validated
  - Existence of the claims in the transparency log was verified offline
  - The signatures were verified against the specified public key

[{"critical":{"identity":{"docker-reference":"index.docker.io/dongjiang1989/node-metrics"},"image":{"docker-manifest-digest":"sha256:c1aa0f2861d2eb744efb8f82a1d7d5f1b74919d1cc6501e799daeac1991fc282"},"type":"cosign container image signature"},"optional":{"Bundle":{"SignedEntryTimestamp":"xxxxxxxxxx","Payload":{"body":"xxxxxxxxxx=","integratedTime":1691032440,"logIndex":29887689,"logID":"c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d"}}}}]

apiVersion: cosignkey.kubeservice.cn/v1
kind: CustomCosignKey
metadata:
  name: test-cosignkey
  namespace: xxxxx
spec:
  authorities:
    key:
    - |
          -----BEGIN PUBLIC KEY-----
          ......
          -----END PUBLIC KEY-----          

type PromClient interface {
	// QueryByNodeIP queries data by node IP.
	QueryByNodeIP(string, string) (string, error)
	// QueryByNodeName queries data by node IP.
	QueryByNodeName(string, string) (string, error)
	// QueryByNodeIPWithOffset queries data by node IP with offset.
	QueryByNodeIPWithOffset(string, string, string) (string, error)
}

//变更为:
type ClientSet interface {
	// QueryByNodeIP queries data by node IP.
	QueryByNodeIP(string, string) (string, error)
	// QueryByNodeName queries data by node IP.
	QueryByNodeName(string, string) (string, error)
	// QueryByNodeIPWithOffset queries data by node IP with offset.
	QueryByNodeIPWithOffset(string, string, string) (string, error)
	// metrics 结果解析
	QueryParse(string, string) (model.Vector, error)
}

type BaseCollector struct {
	metric []typedDesc
	logger log.Logger
}


// NewBaseCollector returns a new Collector exposing base average stats.
func NewBaseCollector(logger log.Logger) (Collector, error) {
	return &BaseCollector{
		metric: []typedDesc{
			{prometheus.NewDesc(namespace+"_avg_1m", "1m base average.", nil, nil), prometheus.GaugeValue},
			{prometheus.NewDesc(namespace+"_avg_5m", "5m base average.", nil, nil), prometheus.GaugeValue},
			{prometheus.NewDesc(namespace+"_avg_1d", "1d base average.", nil, nil), prometheus.GaugeValue},
		},
		logger: logger,
	}, nil
}

func (c *BaseCollector) Update(ch chan<- prometheus.Metric) error {
	loads, err := getData() // 实时指标
	if err != nil {
		return fmt.Errorf("couldn't get load: %w", err)
	}
	for i, load := range loads {
		level.Debug(c.logger).Log("msg", "return load", "index", i, "load", load)
		ch <- c.metric[i].mustNewConstMetric(load)
	}
	return err
}

metrics:
	- name: cpu_usage_avg_5m
      period: 3m
    - name: cpu_usage_max_avg_1h
      period: 15m
    - name: cpu_usage_max_avg_1d
      period: 3h
    - name: mem_usage_avg_5m
      period: 3m
    - name: mem_usage_max_avg_1h
      period: 15m
    - name: mem_usage_max_avg_1d
      period: 3h
    //...

apiVersion: v1
kind: Pod
metadata:
  name: lxcfs-demo
  namespace: default
spec:
  containers:
  - image: busybox
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
    name: busybox
    resources:     #限制了pod资源大小
      requests:
        memory: "64Mi"  
        cpu: "250m"
      limits:
        memory: "128Mi"
        cpu: "500m"
  restartPolicy: Always

dongjiangdeMacBook-Pro:kubernetes $ kubectl exec -it lxcfs-demo  "/bin/sh"
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/ # free -h
              total        used        free      shared  buff/cache   available
Mem:           2.9G      802.9M      117.0M      333.4M        2.0G        1.6G
Swap:        512.0M        1.3M      510.7M
/ # cat /proc/cpuinfo| grep "cpu cores"| uniq   //物理Core数
cpu cores	: 1
/ # cat /proc/cpuinfo| grep "processor"| wc -l  //逻辑Core数
2
/ # 

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: lxcfs
  labels:
    app: lxcfs
spec:
  selector:
    matchLabels:
      app: lxcfs
  template:
    metadata:
      labels:
        app: lxcfs
    spec:
      hostPID: true
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      containers:
      - name: lxcfs
        image: dongjiang1989/lxcfs:v4.0.12
        imagePullPolicy: Always
        securityContext:
          privileged: true
        volumeMounts:
        - name: cgroup
          mountPath: /sys/fs/cgroup
        - name: lxcfs
          mountPath: /var/lib/lxcfs
          mountPropagation: Bidirectional
        - name: usr-local
          mountPath: /usr/local
      volumes:
      - name: cgroup
        hostPath:
          path: /sys/fs/cgroup
      - name: usr-local
        hostPath:
          path: /usr/local
      - name: lxcfs
        hostPath:
          path: /var/lib/lxcfs
          type: DirectoryOrCreate

apiVersion: apps/v1
kind: Deployment
metadata:
  name: lxcfs-admission-webhook-deployment
  namespace: kube-system
  labels:
    app: lxcfs-admission-webhook
spec:
  replicas: 1
  selector:
    matchLabels:
      app: lxcfs-admission-webhook
  template:
    metadata:
      labels:
        app: lxcfs-admission-webhook
    spec:
      serviceAccountName: lxcfs-webhook-serviceaccount
      containers:
        - name: lxcfs-admission-webhook
          image: dongjiang1989/lxcfs-webhook:latest
          imagePullPolicy: Always
          args:
            - -tlsCertFile=/etc/webhook/certs/tls.crt
            - -tlsKeyFile=/etc/webhook/certs/tls.key
            - -alsologtostderr
            - -v=4
            - 2>&1
          resources:
            limits:
              cpu: 500m
              memory: 128Mi
            requests:
              cpu: 10m
              memory: 64Mi
          volumeMounts:
            - mountPath: /etc/webhook/certs/
              name: cert
              readOnly: true
      volumes:
      - name: cert
        secret:
          defaultMode: 420
          secretName: lxcfs-webhook-server-cert

apiVersion: apps/v1
kind: Deployment
metadata:
  name: httpd-test
spec:
  replicas: 1
  selector:
    matchLabels:
      app: httpd-test
  template:
    metadata:
      labels:
        app: httpd-test
    spec:
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      containers:
        - name: httpd
          image: httpd:2.4.32
          imagePullPolicy: Always
          resources:
            requests:
              memory: "256Mi"
              cpu: "1"
            limits:
              memory: "256Mi"
              cpu: "1"

[root@kcs-cpu-test-m-qm2dd /]#  free -h      
              total        used        free      shared  buff/cache   available
Mem:            15G        2.0G        2.5G        4.4M         10G         12G
Swap:            0B          0B          0B
[root@kcs-cpu-test-m-qm2dd /]#  cat /proc/cpuinfo | grep "physical id"
physical id     : 0
physical id     : 0
physical id     : 1
physical id     : 1
[root@kcs-cpu-test-m-qm2dd /]#  cat /proc/cpuinfo | grep processor
processor       : 0
processor       : 1
processor       : 2
processor       : 3

[root@kcs-cpu-test-m-qm2dd /]#  kubectl  get pod | grep "httpd-test"
httpd-test-68b9b9d74f-5tmnh       1/1     Running   0          11m

[root@kcs-cpu-test-m-qm2dd /]#  kubectl exec -it httpd-test-68b9b9d74f-5tmnh "/bin/bash"
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.

root@httpd-test-68b9b9d74f-5tmnh:/usr/local/apache2# free -h
             total       used       free     shared    buffers     cached
Mem:          256M       8.4M       247M       268K         0B       272K
-/+ buffers/cache:       8.1M       247M
Swap:           0B         0B         0B

[root@kcs-cpu-test-m-qm2dd /]#  top
top - 09:31:23 up 15 min,  0 users,  load average: 0.00, 0.00, 0.00
Tasks:   6 total,   1 running,   5 sleeping,   0 stopped,   0 zombie
%Cpu0  :  4.5 us,  2.1 sy,  0.0 ni, 93.4 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem:    262144 total,     9072 used,   253072 free,        0 buffers
KiB Swap:        0 total,        0 used,        0 free.      276 cached Mem

schemaVersion:
mediaType:
config:
  - name: modelx.yaml
    digest: sha256:xxxxxxxx...
    mediaType: "?"
    size: 45
blobs:
  - name: file-a.bin
    digest: sha256:xxxxxxxx...
    mediaType: ""
    size: 18723
  - name: file-b.bin
    digest: sha256:xxxxxxxx...
    mediaType: "?"
    size: 10086
annotaitons:
  description: "some description"

config:
  inputs: {}
  outputs: {}
description: Awesome text generator
framework: pytorch
maintainers:
  - maintainer
tags:
  - modelx
  - demo
annotations: 
  user: dongjiang
  aa: bb
resources:
  cpu: "4"
  gpu:
    nvdia:
      nvdia/gpu: "1"
    memory: 16Gi
task: text-generation
args: 
  - python3
  - -m vllm.entrypoints.openai.api_server 
  - --model ./
  - --port 8000
  - --max-num-batched-tokens 8192
modelFiles:
  - torch.bin
  - tf_output.h5
dependencies:
  - modelbaseA
  - modelbaseB

{
  "name": "k8s-pod-network",
  "cniVersion": "0.3.0",  #必须0.3.0 containernetworking plugin 目前最高版本
  "plugins":
    [
      {
        "type": "calico",
        "log_level": "info",
        "datastore_type": "kubernetes",
        "nodename": "127.0.0.1",
        "ipam": { "type": "host-local", "subnet": "usePodCidr" },
        "policy": { "type": "k8s" },
        "kubernetes": { "kubeconfig": "/etc/cni/net.d/calico-kubeconfig" },
      },
      { 
    	"type": "bandwidth", 
    	"capabilities": {     
			"bandwidth": true   #支持cri-o json配置提交
		},
		/* 以下是对cni插件网络限流操作， capabilities和一下4个配置二选一
		"ingressRate": 123,
      	"ingressBurst": 456,
        "egressRate": 123,
        "egressBurst": 456
        */
	  },
    ]
}

func cmdAdd(args *skel.CmdArgs) error {
	// cni 配置解析
	conf, err := parseConfig(args.StdinData)
	if err != nil {
		return err
	}
    
	//...
	
	// 从配置中活动 ingress Rate和Burst
	if bandwidth.IngressRate > 0 && bandwidth.IngressBurst > 0 {
		// TC TBF 中创建流控规则
		err = CreateIngressQdisc(bandwidth.IngressRate, bandwidth.IngressBurst, hostInterface.Name)
		if err != nil {
			return err
		}
	}
	
	// 从配置中活动 egress Rate和Burst
	if bandwidth.EgressRate > 0 && bandwidth.EgressBurst > 0 {
		// ...
		
		// 对特定本地Device设置出口流控规则
		err = CreateEgressQdisc(bandwidth.EgressRate, bandwidth.EgressBurst, hostInterface.Name, ifbDeviceName)
		if err != nil {
			return err
		}
	}

	return types.PrintResult(result, conf.CNIVersion)
}

apiVersion: v1
kind: Pod
  metadata:
    name: iperf-slow
  annotations:
    kubernetes.io/ingress-bandwidth: 10M
    kubernetes.io/egress-bandwidth: 10M
...

// 配置值在 1k-1p之间
var minRsrc = resource.MustParse("1k")  
var maxRsrc = resource.MustParse("1P")

// 获取pod annotations并传递给 runc
func ExtractPodBandwidthResources(podAnnotations map[string]string) (ingress, egress *resource.Quantity, err error) {
	if podAnnotations == nil {
		return nil, nil, nil
	}
	str, found := podAnnotations["kubernetes.io/ingress-bandwidth"]
	if found {
		ingressValue, err := resource.ParseQuantity(str)
		if err != nil {
			return nil, nil, err
		}
		ingress = &ingressValue
		if err := validateBandwidthIsReasonable(ingress); err != nil {
			return nil, nil, err
		}
	}
	str, found = podAnnotations["kubernetes.io/egress-bandwidth"]
	if found {
		egressValue, err := resource.ParseQuantity(str)
		if err != nil {
			return nil, nil, err
		}
		egress = &egressValue
		if err := validateBandwidthIsReasonable(egress); err != nil {
			return nil, nil, err
		}
	}
	return ingress, egress, nil
}

func cniNamespaceOpts(id string, config *runtime.PodSandboxConfig) ([]cni.NamespaceOpts, error) {
	opts := []cni.NamespaceOpts{
		cni.WithLabels(toCNILabels(id, config)),
		cni.WithCapability(annotations.PodAnnotations, config.Annotations),
	}

	portMappings := toCNIPortMappings(config.GetPortMappings())
	if len(portMappings) > 0 {
		opts = append(opts, cni.WithCapabilityPortMap(portMappings))
	}

	// pod annotations中获得配置，最后传递给cni
	bandWidth, err := toCNIBandWidth(config.Annotations)
	if err != nil {
		return nil, err
	}
	if bandWidth != nil {
		opts = append(opts, cni.WithCapabilityBandWidth(*bandWidth))
	}
	// ...
}

apiVersion: apps/v1
kind: Deployment
metadata:
  name: iperf-server-deployment
  labels:
    app: iperf-server
spec:
  replicas: 1
  selector:
    matchLabels:
      app: iperf-server
  template:
    metadata:
      labels:
        app: iperf-server
      #添加注解
      annotations:
        kubernetes.io/ingress-bandwidth: 1M
        kubernetes.io/egress-bandwidth: 1M
    spec:
      tolerations:
        - key: node-role.kubernetes.io/master
          operator: Exists
          effect: NoSchedule
      containers:
      - name: iperf3-server
        image: dongjiang1989/iperf
        args: ['-s', '-p', '5001']
        ports:
        - containerPort: 5001
          name: server
      terminationGracePeriodSeconds: 0

---
    
apiVersion: apps/v1
kind: Deployment
metadata:
  name: iperf-client
  labels:
    app: iperf-client
spec:
  replicas: 1
  selector:
    matchLabels:
      app: iperf-client
  template:
    metadata:
      labels:
        app: iperf-client
    spec:
      containers:
      - name: iperf-client
        image: dongjiang1989/iperf
        command: ['/bin/sh', '-c', 'sleep 1d']
      terminationGracePeriodSeconds: 0

$ kubectl get pod | grep iperf 
iperf-client-7874c47d95-t7hph              1/1     Running   0               5m58s
iperf-server-deployment-74d94bdd59-dzdl4   1/1     Running   0               5m58s
kubectl exec iperf-client-7874c47d95-t7hph -- iperf -c 10.1.0.173 -p 5001 -i 10 -t 100
------------------------------------------------------------
Client connecting to 10.1.0.173, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[  1] local 10.1.0.172 port 56296 connected with 10.1.0.173 port 5001
[ ID] Interval       Transfer     Bandwidth
[  1] 0.00-10.00 sec  19.7 GBytes  16.9 Gbits/sec
[  1] 10.00-20.00 sec  18.9 GBytes  16.2 Gbits/sec
[  1] 20.00-30.00 sec  20.0 GBytes  17.2 Gbits/sec
[  1] 30.00-40.00 sec  20.4 GBytes  17.5 Gbits/sec
[  1] 40.00-50.00 sec  18.5 GBytes  15.9 Gbits/sec
[  1] 50.00-60.00 sec  19.3 GBytes  16.5 Gbits/sec
[  1] 60.00-70.00 sec  17.6 GBytes  15.1 Gbits/sec
[  1] 70.00-80.00 sec  17.1 GBytes  14.7 Gbits/sec
[  1] 80.00-90.00 sec  18.4 GBytes  15.8 Gbits/sec
[  1] 90.00-100.00 sec  15.1 GBytes  13.0 Gbits/sec
[  1] 0.00-100.00 sec   185 GBytes  15.9 Gbits/sec

$ kubectl get pod | grep iperf
iperf-clients-rcsh6                        1/1     Running   0          7h7m
iperf-server-deployment-59675c8f78-g52pm   1/1     Running   0          6h52m

$ kubectl exec iperf-clients-rcsh6 -- iperf -c 10.1.0.170 -p 5001 -i 10 -t 100
------------------------------------------------------------
Client connecting to 10.1.0.170, TCP port 5001
TCP window size: 45.0 KByte (default)
------------------------------------------------------------
[  1] local 10.1.0.170 port 54652 connected with 10.1.0.170 port 5001
[ ID] Interval       Transfer     Bandwidth
[  1] 0.00-10.00 sec  3.50 MBytes  2.94 Mbits/sec
[  1] 10.00-20.00 sec  2.25 MBytes  1.89 Mbits/sec
[  1] 20.00-30.00 sec  2.04 MBytes  1.71 Mbits/sec
[  1] 30.00-40.00 sec   892 KBytes   731 Kbits/sec
[  1] 40.00-50.00 sec   954 KBytes   781 Kbits/sec
[  1] 50.00-60.00 sec  1.36 MBytes  1.14 Mbits/sec
[  1] 60.00-70.00 sec  1.18 MBytes   993 Kbits/sec
[  1] 70.00-80.00 sec  87.1 KBytes  71.4 Kbits/sec
[  1] 80.00-90.00 sec  0.000 Bytes  0.000 bits/sec
[  1] 90.00-100.00 sec  2.97 MBytes  2.50 Mbits/sec
[  1] 0.00-100.69 sec  15.5 MBytes  1.29 Mbits/sec

apiVersion: custom.xxx.com/v1
kind: CustomLimitRange
metadata:
  name: test-rangelimit
spec:
  limitrange:
    type: pod      # 对pod类型限制，以后扩展到 contianer类型、ingress类型、service类型
    max:           # max和min是限制的上下线，如果pod自定义的值不在其中，ValidatingAdmissionWebhook校验报错
      ingress-bandwidth: "1G"
      egress-bandwidth: "1G"
    min:
      ingress-bandwidth: "10M"
      egress-bandwidth: "10M"
    default:          # 定义了default，如果pod annotation为空，MutatingAdmissionWebhook自动注入此数据；未定义default，不作强注入操作
      ingress-bandwidth: "128M"
      egress-bandwidth: "128M"

# Pod
apiVersion: v1
kind: Pod
metadata:
  name: xxxx
  annotations:
    kubernetes.io/ingress-bandwidth: 1M
    kubernetes.io/egress-bandwidth: 1M
...


# Deployment
...
 spec:
  template:
    metadata:
      #添加注解
      annotations:
        kubernetes.io/ingress-bandwidth: 1M
        kubernetes.io/egress-bandwidth: 1M
...

// MountInfo contains the information needed by Kata to consume a host block device and mount it as a filesystem inside the guest VM.
type MountInfo struct {
	// The type of the volume (ie. block)
	VolumeType string `json:"volume-type"`
	// The device backing the volume.
	Device string `json:"device"`
	// The filesystem type to be mounted on the volume.
	FsType string `json:"fstype"`
	// Additional metadata to pass to the agent regarding this volume.
	Metadata map[string]string `json:"metadata,omitempty"`
	// Additional mount options.
	Options []string `json:"options,omitempty"`
}

mountInfo := &volume.MountInfo{
				VolumeType: "block",  // 设备类型
    				Device:     dev/sdd,  // 块设备路径
    				FsType:     ext4,     // 文件系统类型
			}
volume.Add("/run/kata-containers/shared/direct-volumes/volume-path(base64加密)/", mountInfo) 

import (
    "encoding/json"
    volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
    "google.golang.org/grpc/status"
    klog "k8s.io/klog/v2"
)

// NodePublishVolume 中发布
func AddDirectVolume(volumePath, device, fsType string) error {
    mountInfo := &volume.MountInfo{
        VolumeType: "block",
        Device:     device,
        FsType:     fsType,
    }

    mi, err := json.Marshal(mountInfo)
    if err != nil {
        klog.Errorf("addDirectVolume - json.Marshal failed: ", err.Error())
        return status.Errorf(codes.Internal, "json.Marshal failed: %s", err.Error())
    }
    
    if err := volume.Add(volumePath, string(mi)); err != nil { 
        klog.Errorf("addDirectVolume - add direct volume failed: ", err.Error())
        return status.Errorf(codes.Internal, "add direct volume failed: %s", err.Error())
    }

    klog.Infof("add direct volume done: %s%s", volumePath, string(mi))
    return nil
}

//  NodeUnpublishVolume 中remove掉
	if err := volume.Remove(targetPath); err != nil {
		log.Errorf("NodeUnpublishVolume: kata direct volume remove failed: %s", err.Error())
	}

$ kata-runtime direct-volume add --volume-path [volumePath] --mount-info [mountinfo.json]

$ kata-runtime direct-volume remove --volume-path [volumePath] --mount-info [mountinfo.json]

---
apiVersion: v1
kind: Pod
metadata:
  name: app
spec:
  runtime-class: kata-qemu  # 方式一：通过设置runtime-class，使用kata 运行时； 其下面的卷都是直通卷
  containers:
  - name: app
    image: centos
    command: ["/bin/sh"]
    args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"]
    volumeMounts:
    - name: persistent-storage
      mountPath: /data
  volumes:
  - name: persistent-storage
    persistentVolumeClaim:
      claimName: ebs-claim
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  annotations:
    skip-hostmount: "true"  # 方式二：通知csi为卡通直通卷
  name: ebs-claim
spec:
  accessModes:
    - ReadWriteOncePod
  volumeMode: Filesystem
  storageClassName: ebs-sc
  resources:
    requests:
      storage: 1Gi
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: ebs-sc
provisioner: ebs.csi.aws.com # 方式三：特定的csi driver实现 kata direct assigned volume
volumeBindingMode: WaitForFirstConsumer
parameters:
  csi.storage.k8s.io/fstype: ext4

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: test
  namespace: test
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
  storageClassName: csi-cephfs-sc
  volumeMode: Filesystem

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: csi-rbd-sc
parameters:
  clusterID: ceph01
  imageFeatures: layering
  imageFormat: "2"
  mounter: rbd
  pool: kubernetes
provisioner: rbd.csi.ceph.com
reclaimPolicy: Delete
volumeBindingMode: Immediate

apiVersion: storage.k8s.io/v1
kind: VolumeAttachment
metadata:
  name: csi-123456
spec:
  attacher: cephfs.csi.ceph.com
  nodeName: 172.1.1.10
  source:
    persistentVolumeName: pvc-123456
status:
  attached: true

apiVersion: storage.k8s.io/v1
kind: CSINode
metadata:
  name: 172.1.1.10
spec:
  drivers:
  - name: cephfs.csi.ceph.com
    nodeID:  172.1.1.10
    topologyKeys: null
  - name: rbd.csi.ceph.com
    nodeID:  172.1.1.10
    topologyKeys: null

dongjiangdeMacBook-Pro:~ $ docker help run | grep -E 'bps|IO'
Usage:  docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
      --blkio-weight uint16            Block IO (relative weight), between 10 and 1000, or 0 to disable (default 0)
      --blkio-weight-device list       Block IO weight (relative device weight) (default [])
      --device-read-bps list           Limit read rate (bytes per second) from a device (default [])
      --device-read-iops list          Limit read rate (IO per second) from a device (default [])
      --device-write-bps list          Limit write rate (bytes per second) to a device (default [])
      --device-write-iops list         Limit write rate (IO per second) to a device (default [])

# Pod
apiVersion: v1
kind: Pod
metadata:
  name: xxxx
  annotations:
    io.kubernetes.container.blkio: '{"weight":200,"weight_device":[{"device":"rootfs","value":"200k"}],"device_read_bps":[{"device":"/dev/sda1","value":"20m"}],"device_write_bps":[{"device":"rootfs","value":"20m"}],"device_read_iops":[{"device":"rootfs","value":"200"}]"device_write_iops":[{"device":"rootfs","value":"300"}]}'
...

// +build linux 
// 只是对liunx系统生效
func UpdateBlkio(containerId string, docker libdocker.Interface) (err error) {
	info, err := docker.InspectContainer(containerId) //获得docker inspect info
	
	blkio := Blkio{}
	err = json.Unmarshal([]byte(blkiolable), &blkio) //添加blkio设置
	if err != nil {
		return fmt.Errorf("failed to unmarshal blkio config,%s, sandboxID:%s, containerId:%s", err.Error(), sandboxID, containerId)
	}

	//...
	blkioResource, err := getBlkioResource(&blkio, containerRoot) //获得 blkioResource object
	if err != nil {
		return fmt.Errorf("getBlkioResource failed. sandboxID:%s, containerId:%s, %v", sandboxID, containerId, err.Error())
	}

	cg := &configs.Cgroup{
		Path:      cpath,
		Resources: &blkioResource,
	}
	err = blkioSubsystem.Set(cpath, cg)  // blkio设置到cgroup
	if err != nil {
		return fmt.Errorf("blkioSubsystem.Set failed. sandboxID:%s, containerId:%s, %v", sandboxID, containerId, err.Error())
	}
	glog.V(4).Infof("set Blkio cgroup success. sandboxID:%s, containerId:%s, cgroup path:%v, cgroup:%+v", sandboxID, containerId, cpath, cgroupToString(cg))
	return nil
}

# Pod
apiVersion: v1
kind: Pod
metadata:
  name: xxxx
  annotations:
    qos.volume.storage.cloud.cmss.com: >-
        {"pvc": "snap-03", "iops": {"read": 2000, "write": 1000}, "bps":
        {"read": 1000000, "write": 1000000}}        
...

 echo "<block_device_maj:min> <value>" > /sys/fs/cgroup/blkio/kubepods/pod<UID>/blkio.throttle.read_iops_device

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
    name: csi-local
provisioner: local.csi.cmsss.com
parameters:
    volumeType: LVM
    vgName: volumegp
    fsType: ext4
    lvmType: "striping"
    readBPS: 1M  # read 此类卷的带宽是1MB/s
    writeBPS: 100K # 写 此类卷的带宽是100KB/s
    readIOPS: 2000  # read 此类卷的tps是2000
    writeIOPS: 1000  # write 此类卷的tps是1000
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: true

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: lvm-pvc
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 2Gi
  storageClassName: csi-local

apiVersion: apps/v1
kind: Deployment
metadata:
  name: deployment-lvm
  labels:
    app: nginx
spec:
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.15.4
        volumeMounts:
          - name: lvm-pvc
            mountPath: "/data"
      volumes:
        - name: lvm-pvc
          persistentVolumeClaim:
            claimName: lvm-pvc

func SetVolumeIOLimit(devicePath string, req *csi.NodePublishVolumeRequest) error {
	readIOPS := req.VolumeContext["readIOPS"]
	writeIOPS := req.VolumeContext["writeIOPS"]
	readBPS := req.VolumeContext["readBPS"]
	writeBPS := req.VolumeContext["writeBPS"]

	// IOlimit 值解析
	readBPSInt, err := getBpsLimt(readBPS)
	if err != nil {
		log.Errorf("Volume(%s) Input Read BPS Limit format error: %s", req.VolumeId, err.Error())
		return err
	}
	writeBPSInt, err := getBpsLimt(writeBPS)
	if err != nil {
		log.Errorf("Volume(%s) Input Write BPS Limit format error: %s", req.VolumeId, err.Error())
		return err
	}
	readIOPSInt := 0
	if readIOPS != "" {
		readIOPSInt, err = strconv.Atoi(readIOPS)
		if err != nil {
			log.Errorf("Volume(%s) Input Read IOPS Limit format error: %s", req.VolumeId, err.Error())
			return err
		}
	}
	writeIOPSInt := 0
	if writeIOPS != "" {
		writeIOPSInt, err = strconv.Atoi(writeIOPS)
		if err != nil {
			log.Errorf("Volume(%s) Input Write IOPS Limit format error: %s", req.VolumeId, err.Error())
			return err
		}
	}

	// 获得 Device major/minor 值
	majMinNum := getMajMinDevice(devicePath)
	if majMinNum == "" {
		log.Errorf("Volume(%s) Cannot get major/minor device number: %s", req.VolumeId, devicePath)
		return errors.New("Volume Cannot get major/minor device number: " + devicePath + req.VolumeId)
	}

	// 获得pod uid
	podUID := req.VolumeContext["csi.storage.k8s.io/pod.uid"]
	if podUID == "" {
		log.Errorf("Volume(%s) Cannot get poduid and cannot set volume limit", req.VolumeId)
		return errors.New("Cannot get poduid and cannot set volume limit: " + req.VolumeId)
	}
	// 写具体的pod blkio文件
	podUID = strings.ReplaceAll(podUID, "-", "_")
	podBlkIOPath := filepath.Join("/sys/fs/cgroup/blkio/kubepods.slice/kubepods-besteffort.slice", "kubepods-besteffort-pod"+podUID+".slice")
	if !IsHostFileExist(podBlkIOPath) {
		podBlkIOPath = filepath.Join("/sys/fs/cgroup/blkio/kubepods.slice/kubepods-burstable.slice", "kubepods-besteffort-pod"+podUID+".slice")
	}
	if !IsHostFileExist(podBlkIOPath) {
		log.Errorf("Volume(%s), Cannot get pod blkio/cgroup path: %s", req.VolumeId, podBlkIOPath)
		return errors.New("Cannot get pod blkio/cgroup path: " + podBlkIOPath)
	}

	// 设置具体pod blkio文件值
	if readIOPSInt != 0 {
		err := writeIoLimit(majMinNum, podBlkIOPath, "blkio.throttle.read_iops_device", readIOPSInt)
		if err != nil {
			return err
		}
	}
	if writeIOPSInt != 0 {
		err := writeIoLimit(majMinNum, podBlkIOPath, "blkio.throttle.write_iops_device", writeIOPSInt)
		if err != nil {
			return err
		}
	}
	if readBPSInt != 0 {
		err := writeIoLimit(majMinNum, podBlkIOPath, "blkio.throttle.read_bps_device", readBPSInt)
		if err != nil {
			return err
		}
	}
	if writeBPSInt != 0 {
		err := writeIoLimit(majMinNum, podBlkIOPath, "blkio.throttle.write_bps_device", writeBPSInt)
		if err != nil {
			return err
		}
	}
	log.Infof("Seccessful Set Volume(%s) IO Limit: readIOPS(%d), writeIOPS(%d), readBPS(%d), writeBPS(%d)", req.VolumeId, readIOPSInt, writeIOPSInt, readBPSInt, writeBPSInt)
	return nil
}

$ docker build -f hack/local/Dockerfile .

$ fdisk -l
Disk /dev/sdc: 68.7 GB, 68719476736 bytes, 134217728 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes

$ fdisk /dev/sdc
Command (m for help): p

Disk /dev/sdc: 68.7 GB, 68719476736 bytes, 134217728 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x72a4d30c

   Device Boot      Start         End      Blocks   Id  System

Command (m for help): n
Partition type:
   p   primary (0 primary, 0 extended, 4 free)
   e   extended
Select (default p): p
Partition number (1-4, default 1): 1
First sector (2048-134217727, default 2048): 
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-134217727, default 134217727): 64217727        
Partition 1 of type Linux and of size 30.6 GiB is set

Command (m for help): t
Selected partition 1
Hex code (type L to list all codes): 8e
Changed type of partition 'Linux' to 'Linux LVM'

Command (m for help): w
The partition table has been altered!

$ lsblk 
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sdb      8:16   0  100G  0 disk 
|-sdb4   8:20   0   25G  0 part 
`-sdb2   8:18   0   75G  0 part 
sr0     11:0    1  506K  0 rom  
sdc      8:32   0   64G  0 disk 
`-sdc1   8:33   0 30.6G  0 part 
sda      8:0    0  100G  0 disk 
`-sda1   8:1    0  100G  0 part /

$ pvcreate /dev/sdc1 
  Physical volume "/dev/sdc1" successfully created.
$ vgcreate volumegroup1 /dev/sdc1
  Volume group "volumegroup1" successfully created
$ vgdisplay 
  --- Volume group ---
  VG Name               volumegroup1
  System ID             
  Format                lvm2
  Metadata Areas        1
  Metadata Sequence No  2
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                1
  Open LV               1
  Max PV                0
  Cur PV                1
  Act PV                1
  VG Size               <30.62 GiB
  PE Size               4.00 MiB
  Total PE              7838
  Alloc PE / Size       512 / 2.00 GiB
  Free  PE / Size       7326 / <28.62 GiB
  VG UUID               V6TVTh-AcIi-hLmR-bozc-9QeA-EBnU-Mhhd6y

$ kubectl create -f ./deploy/local/provisioner.yaml

$ kubectl create -f ./deploy/local/plugin.yaml

$ kubectl create -f ./examples/storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: csi-lvm
provisioner: local.csi.ecloud.cmss.com
parameters:
    vgName: volumegroup1
    fsType: ext4
    pvType: localdisk
    nodeAffinity: "false"
    readIOPS: "2000"
    writeIOPS: "1000"
    readBPS: "10000"
    writeBPS: "5000"
reclaimPolicy: Delete

$ kubectl create -f ./examples/pvc.yaml
$ kubectl create -f ./examples/deploy.yaml

$ kubectl get pvc
NAME      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
lvm-pvc   Bound    lvm-29def33c-8dae-482f-8d64-c45e741facd9   2Gi        RWO            csi-lvm        3h37m

$ kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM             STORAGECLASS   REASON   AGE
lvm-29def33c-8dae-482f-8d64-c45e741facd9   2Gi        RWO            Delete           Bound    default/lvm-pvc   csi-lvm                 3h38m

$ kubectl get pod | grep deployment-lvm
deployment-lvm-57bc9bcd64-j7r9x   1/1     Running   0          77s

$ kubectl exec -ti deployment-lvm-57bc9bcd64-j7r9x   sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
# df -h | grep data
/dev/mapper/volumegroup1-lvm--9e30e658--5f85--4ec6--ada2--c4ff308b506e  2.0G  6.0M  1.8G   1% /data

$ kubectl describe pod deployment-lvm-57bc9bcd64-j7r9x | grep Node:
Node:         kcs-cpu-test-m-8mzmj/172.16.0.67

$ ifconfig | grep 172.16.0.67
        inet 172.16.0.67  netmask 255.255.0.0  broadcast 172.16.255.255
  
$ mount | grep volumegroup
/dev/mapper/volumegroup1-lvm--9e30e658--5f85--4ec6--ada2--c4ff308b506e on /var/lib/kubelet/pods/c06d5521-3d9c-4517-bdc2-e6df34b9e8f1/volumes/kubernetes.io~csi/lvm-9e30e658-5f85-4ec6-ada2-c4ff308b506e/mount type ext4 (rw,relatime,data=ordered)
/dev/mapper/volumegroup1-lvm--9e30e658--5f85--4ec6--ada2--c4ff308b506e on /var/lib/paascontainer/kubelet/pods/c06d5521-3d9c-4517-bdc2-e6df34b9e8f1/volumes/kubernetes.io~csi/lvm-9e30e658-5f85-4ec6-ada2-c4ff308b506e/mount type ext4 (rw,relatime,data=ordered)

$ pwd
/sys/fs/cgroup/blkio/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podc06d5521_3d9c_4517_bdc2_e6df34b9e8f1.slice

$ cat blkio.throttle.read_bps_device 
253:1 10000
$ cat blkio.throttle.write_bps_device 
253:1 5000
$ cat blkio.throttle.write_iops_device 
253:1 1000
$ cat blkio.throttle.read_iops_device 
253:1 2000

$ kubectl label nodes docker-desktop key=value
node/docker-desktop labeled

$ kubectl get nodes --show-labels             
NAME             STATUS   ROLES                  AGE    VERSION   LABELS
docker-desktop   Ready    control-plane,master   109d   v1.22.5   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,key=value,kubernetes.io/arch=amd64,kubernetes.io/hostname=docker-desktop,kubernetes.io/os=linux,node-role.kubernetes.io/control-plane=,node-role.kubernetes.io/master=,node.kubernetes.io/exclude-from-external-load-balancers=

apiVersion: v1
kind: Pod
metadata:
  labels:
    app: busybox
  name: test
spec:
  containers:
  - command:
    - sleep
    - "1d"
    image: busybox
    name: test
  nodeSelector:  
    # 通过nodeSelector强制进行亲和部署，对于有 key: value label的node没有资源等限制，Pod会直接失败
    key: value

apiVersion: v1
kind: Pod
metadata:
  name: node-affinity
  labels:
    app: node-affinity-test
spec:
  containers:
  - name: node-affinity
    image: nginx
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution: # 硬策略
        nodeSelectorTerms:
        - matchExpressions:
          - key: kubernetes.io/os
            operator: In  #可以有 In、NotIn、Gt、Lt、Exists、DoesNotExist
            values:
            - Linux
        - matchExpressions:
          - key: kubernetes.io/os
            operator: In
            values:
            - Window
      preferredDuringSchedulingIgnoredDuringExecution:  # 软策略
      - weight: 1
        preference:
          matchExpressions:
          - key: key
            operator: In
            values:
            - value
            - value1

$ kubectl get node
NAME             STATUS   ROLES                  AGE    VERSION
docker-desktop   Ready    control-plane,master   109d   v1.22.5

$ kubectl taint nodes docker-desktop key=value:PreferNoSchedule
node/docker-desktop tainted

$ kubectl get node docker-desktop -o go-template --template {{.spec.taints}}
[map[effect:PreferNoSchedule key:key value:value]]  

tolerations:
- key: "keu"
  operator: "Equal"
  value: "value"
  effect: "PreferNoSchedule"

tolerations:
- key: "key"
  operator: "Exists"
  effect: "PreferNoSchedule"
  tolerationSeconds: 600 # 节点不健康，600秒后再驱逐

apiVersion: v1
kind: Namespace
metadata:
  name: test-demo
  labels:
    # namespace级生效，对本namespace开启
    namespace-node-affinity: enabled

apiVersion: v1
kind: ConfigMap
metadata:
  name: namespace-node-affinity-webhook
  namespace: kube-system  # 这个namespace是 webhook所在的namespace
data:
  # namespace 名称: test-demo、default
  setting-namespace-name: |  
    nodeSelectorTerms:  #做 nodeAffinity 下的 硬策略
      - matchExpressions:
        - key: "kubernetes.io/os"
          operator: In
          values:
          - "linux"
    tolerations:  #做 tolerations 策略
      - key: "example-key"
        operator: "Exists"
        effect: "NoSchedule"
    excludedLabels:  #如果匹配其中全部key-value 可跳过强绑定
      namespace-node-affinity.cmss.com: disabled

# Pod
apiVersion: v1
kind: Pod
metadata:
  name: xxxx
  annotations:
    namespace-node-affinity.cmss.com: disabled
...


# Deployment
...
 spec:
  template:
    metadata:
      #添加注解
      annotations:
        namespace-node-affinity.cmss.com: disabled
...