Lixueduan | 李学端修改

1
2

free -h
ps -eo pid,ppid,comm,rss,%mem,etime --sort=-rss | head -20

1
2
3
4
5

COMMAND RSS (GiB)
kubelet 7.50
kube-apiserver 1.50
java (Keycloak) 0.85
dcgm-exporter 0.43

1
2
3

NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
kubectl get --raw "/api/v1/nodes/${NODE}/proxy/metrics" \
 | grep -E '^(go_memstats_heap_alloc_bytes|process_resident_memory_bytes) '

1
2
3
4
5

NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
kubectl --request-timeout=30s get --raw "/api/v1/nodes/${NODE}/proxy/debug/pprof/heap" \
 > kubelet-heap.pb.gz

go tool pprof -top -inuse_space -nodecount=30 kubelet-heap.pb.gz

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

File: kubelet
Type: inuse_space
Showing nodes accounting for 6007.96MB, 97.45% of 6165.42MB total

 flat flat% cum cum%
 3643.23MB 59.09% 5031.88MB 81.61% context.(*cancelCtx).propagateCancel
 1388.65MB 22.52% 1388.65MB 22.52% context.(*cancelCtx).Done
 975.57MB 15.82% 1225.96MB 19.88% context.withCancel
 0 0% 4612.69MB 74.82% k8s.io/apimachinery/pkg/util/wait.PollUntilContextTimeout
 0 0% 4625.76MB 75.03% k8s.io/kubernetes/pkg/kubelet.(*Kubelet).SyncPod
 0 0% 4612.69MB 74.82% k8s.io/kubernetes/pkg/kubelet/volumemanager.(*volumeManager).WaitForAttachAndMount
 0 0% 1225.96MB 19.88% k8s.io/kubernetes/pkg/kubelet.(*podWorkers).startPodSync

1
2
3
4

if status.ctx == nil || status.ctx.Err() == context.Canceled {
 status.ctx, status.cancelFn = context.WithCancel(context.Background())
}
ctx = status.ctx

1

ctx, status.cancelFn = context.WithCancel(parentCtx)

1
2
3
4
5

if (becameTerminating || wasGracePeriodShortened) && status.cancelFn != nil {
 updateLogger.V(3).Info("Cancelling current pod sync", "workType", status.WorkType())
 status.cancelFn()
 return
}

1
2
3
4
5

type podSyncStatus struct {
 ctx context.Context
 cancelFn context.CancelFunc
 // 其余字段略
}

1
2
3
4
5

-ctx, status.cancelFn = context.WithCancel(parentCtx)
+if status.ctx == nil || status.ctx.Err() != nil {
+ status.ctx, status.cancelFn = context.WithCancel(context.WithoutCancel(parentCtx))
+}
+ctx = status.ctx

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

vllm serve /model \
 --served-model-name kv-demo \
 --dtype half \
 --max-model-len 8192 \
 --max-num-seqs 1 \
 --max-num-batched-tokens 8192 \
 --enforce-eager \
 --enable-prefix-caching \
 --kv-cache-memory-bytes 268435456 \
 --port 8000

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

router:
 modelServers:
 type: vllm
 protocol: http
 matchLabels:
 app: kv-demo
 targetPorts:
 - number: 8000
 inferencePool:
 create: true
 failureMode: FailClose
 epp:
 replicas: 1
 flags:
 refresh-metrics-interval: 1s
 proxy:
 enabled: true
 proxyType: envoy
 mode: sidecar
 failOpen: false
 tokenizer:
 enabled: false

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

apiVersion: llm-d.ai/v1
kind: EndpointPickerConfig
plugins:
- type: utilization-detector
 parameters:
 metricsStalenessThreshold: 5s
- type: single-profile-handler
- type: approx-prefix-cache-producer
 parameters:
 autoTune: false
 blockSizeTokens: 64
 maxPrefixTokensToMatch: 8192
 lruCapacityPerServer: 4096
- type: prefix-cache-scorer
- type: max-score-picker
schedulingProfiles:
- name: default
 plugins:
 - pluginRef: prefix-cache-scorer
 weight: 1.0
 - pluginRef: max-score-picker

1
2
3
4
5

第一次 messages[0].content:
 {context_prefix}\nQuestion: Summarize the observations.

第二次 messages[0].content:
 {context_prefix}\nQuestion: Which measurements should engineers collect next?

1
2
3
4
5
6
7
8

{
 "model": "kv-demo",
 "messages": [{"role": "user", "content": "这里填入展开后的上下文前缀与问题"}],
 "max_tokens": 32,
 "temperature": 0,
 "stream": true,
 "stream_options": {"include_usage": true}
}

1
2
3
4
5

vllm:request_success_total
vllm:prefix_cache_hits_total
vllm:prefix_cache_queries_total
vllm:request_prefill_kv_computed_tokens_sum
vllm:request_prefill_kv_computed_tokens_count

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

apiVersion: scheduling.k8s.io/v1beta1
kind: Workload
metadata:
 name: training
spec:
 podGroupTemplates:
 - name: workers
 schedulingPolicy:
 gang:
 minCount: 2

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

apiVersion: scheduling.k8s.io/v1beta1
kind: PodGroup
metadata:
 name: training-workers
spec:
 workloadRef:
 workloadName: training
 templateName: workers
 schedulingPolicy:
 gang:
 minCount: 2

1
2
3

spec:
 schedulingGroup:
 podGroupName: training-workers

1
2
3
4

kubectl get node

NAME STATUS ROLES AGE VERSION
node-1 Ready control-plane 9m20s v1.37.0

1
2

mkdir -p /root/k8s-137-manifests-backup
cp -a /etc/kubernetes/manifests/*.yaml /root/k8s-137-manifests-backup/

1
2
3
4
5
6
7

spec:
 containers:
 - command:
 - kube-apiserver
 - --feature-gates=GenericWorkload=true
 - --runtime-config=scheduling.k8s.io/v1beta1=true
 # 其他已有参数省略

1
2

kubectl get pods -n kube-system -l tier=control-plane
ps -ef | grep -E '[k]ube-(apiserver|controller-manager|scheduler)'

1
2
3

kube-apiserver --feature-gates=GenericWorkload=true --runtime-config=scheduling.k8s.io/v1beta1=true ...
kube-controller-manager --feature-gates=GenericWorkload=true ...
kube-scheduler --feature-gates=GenericWorkload=true ...

1

kubectl api-resources --api-group=scheduling.k8s.io

1
2
3
4

NAME SHORTNAMES APIVERSION NAMESPACED KIND
podgroups scheduling.k8s.io/v1beta1 true PodGroup
priorityclasses pc scheduling.k8s.io/v1 false PriorityClass
workloads scheduling.k8s.io/v1beta1 true Workload

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

kubectl create namespace gang-demo

kubectl apply -f - <<'EOF'
apiVersion: v1
kind: Pod
metadata:
 name: blocker
 namespace: gang-demo
spec:
 containers:
 - name: blocker
 image: busybox:1.37.0
 command: ["sh", "-c", "sleep 3600"]
 resources:
 requests:
 cpu: "2"
EOF

kubectl get pods -n gang-demo

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

for i in 0 1; do
 kubectl apply -f - <<EOF
apiVersion: v1
kind: Pod
metadata:
 name: plain-${i}
 namespace: gang-demo
spec:
 containers:
 - name: worker
 image: busybox:1.37.0
 command: ["sh", "-c", "sleep 3600"]
 resources:
 requests:
 cpu: "3"
EOF
done

1
2
3
4
5
6

kubectl get pods -n gang-demo

NAME READY STATUS RESTARTS AGE
blocker 1/1 Running 0 ...
plain-0 1/1 Running 0 ...
plain-1 0/1 Pending 0 ...

1

kubectl delete pod -n gang-demo plain-0 plain-1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

kubectl apply -f - <<'EOF'
apiVersion: scheduling.k8s.io/v1beta1
kind: PodGroup
metadata:
 name: workers
 namespace: gang-demo
spec:
 schedulingPolicy:
 gang:
 minCount: 2
EOF

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

for i in 0 1; do
 kubectl apply -f - <<EOF
apiVersion: v1
kind: Pod
metadata:
 name: gang-${i}
 namespace: gang-demo
spec:
 schedulingGroup:
 podGroupName: workers
 containers:
 - name: worker
 image: busybox:1.37.0
 command: ["sh", "-c", "sleep 3600"]
 resources:
 requests:
 cpu: "3"
EOF
done

1
2
3
4
5
6
7

kubectl get pods -n gang-demo \
 -o custom-columns=NAME:.metadata.name,PHASE:.status.phase,NODE:.spec.nodeName

NAME PHASE NODE
blocker Running node-1
gang-0 Pending <none>
gang-1 Pending <none>

1
2
3
4
5

kubectl describe podgroup workers -n gang-demo

Type: PodGroupInitiallyScheduled
Status: False
Reason: Unschedulable

1
2
3

kubectl delete pod blocker -n gang-demo

kubectl get pods -n gang-demo

1
2
3

NAME READY STATUS RESTARTS AGE
gang-0 1/1 Running 0 ...
gang-1 1/1 Running 0 ...

1

kubectl delete namespace gang-demo

1

return {"queue_depth", tostring(redis.call("LLEN", "jobs"))}

1

redis_script_values{key="queue_depth",namespace="hpa-zero",queue="jobs"} 40

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

rules:
 default: false
 external:
 - seriesQuery: 'redis_script_values{key="queue_depth",namespace!=""}'
 resources:
 overrides:
 namespace: {resource: namespace}
 name:
 matches: ^redis_script_values$
 as: queue_depth
 metricsQuery: >-
 sum(<<.Series>>{key="queue_depth",<<.LabelMatchers>>}
 and on(job,instance) (up{job="redis"} == 1)
 and on(job,instance) (redis_up == 1)) by (namespace,queue)

1
2

kubectl get --raw \
 '/apis/external.metrics.k8s.io/v1beta1/namespaces/hpa-zero/queue_depth?labelSelector=queue%3Djobs'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

kubectl apply -f - <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
 name: queue-worker
 namespace: hpa-zero
spec:
 replicas: 1
 selector:
 matchLabels: {app: queue-worker}
 template:
 metadata:
 labels: {app: queue-worker}
 spec:
 terminationGracePeriodSeconds: 5
 containers:
 - name: worker
 image: redis:7.4.2-alpine
 command: ["sh", "-ec"]
 args:
 - |
 while true; do
 task=$(redis-cli -h redis --raw LPOP jobs)
 if [ -n "$task" ]; then
 echo "$(date -u +%FT%TZ) processing $task"
 sleep 2
 redis-cli -h redis RPUSH completed "$task" >/dev/null
 echo "$(date -u +%FT%TZ) completed $task"
 else
 sleep 1
 fi
 done
 resources:
 requests: {cpu: 50m, memory: 32Mi}
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
 name: queue-worker
 namespace: hpa-zero
spec:
 scaleTargetRef:
 apiVersion: apps/v1
 kind: Deployment
 name: queue-worker
 minReplicas: 0
 maxReplicas: 4
 metrics:
 - type: External
 external:
 metric:
 name: queue_depth
 selector:
 matchLabels: {queue: jobs}
 target:
 type: Value
 value: "10"
 behavior:
 scaleDown:
 stabilizationWindowSeconds: 30
EOF

1

kubectl get hpa -n hpa-zero -w

1

kubectl get deploy,pods -n hpa-zero

1

kubectl describe hpa queue-worker -n hpa-zero

1
2
3

Type: ScaledToZero
Status: True
Reason: ScaledToZero

1
2

kubectl exec -n hpa-zero deploy/redis -- redis-cli EVAL \
 'for i=1,40 do redis.call("RPUSH","jobs","task-"..i) end return redis.call("LLEN","jobs")' 0

1
2
3
4

$ kubectl exec -n hpa-zero deploy/redis -- redis-cli LLEN jobs
0
$ kubectl exec -n hpa-zero deploy/redis -- redis-cli LLEN completed
40

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

Goal:
修复 Controller 在临时 API 错误后无法正常恢复的问题。

Scope:
- 检查并修改相关 Reconciler
- 补充对应测试

Constraints:
- 不修改 CRD Schema 和现有公共接口
- 优先复用项目已有的错误处理与重试机制
- 不引入新的重试框架
- 保持 Reconcile 幂等

Acceptance Criteria:
- 临时错误不会被误判为调谐成功
- API 恢复后，资源最终达到期望状态
- 重复调谐不会创建重复资源

Verification:
- 单元测试：验证临时错误的处理和返回行为
- 集成测试：运行实际调谐循环，验证临时错误恢复后资源能够收敛
- 幂等性测试：验证重复调谐不会创建重复资源
- 执行项目规定的 Build / Test / Lint 检查

1
2
3
4
5
6
7
8

先不要修改代码。

阅读相关实现后给出 Plan：
1. 当前逻辑是什么
2. 准备采用什么方案，为什么
3. 需要修改哪些文件
4. 有哪些假设、风险，以及对现有行为的影响
5. 准备如何验证

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

# Project

This is a Go project based on Kubernetes controller-runtime.

## Commands

- `make build`
- `make test`
- `make lint`

## Conventions

- Follow existing controller/reconciler patterns.
- Keep public APIs backward compatible.
- Add tests for new reconciliation logic.
- Run gofmt on modified Go files.

## Restrictions

- Do not modify CRD APIs unless explicitly requested.
- Do not update dependencies without explicit approval.

1
2
3
4
5
6
7
8
9

先阅读：
internal/controller/foo_controller.go
internal/controller/foo_controller_test.go

另外参考：
internal/controller/bar_controller.go

其中 bar_controller.go 的错误处理和 Event 记录方式
可以作为本次实现参考。

1
2
3

make build
make test
make lint

1
2
3
4

Spec → 需求范围和验收标准是否明确
Design → 方案和关键决策是否已经确认
Coding → 约定的变更及对应测试是否已经完成
Verify → 必要检查是否执行，验收结果是否有记录

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

当前阶段：验证
当前任务：修复临时 API 错误后无法恢复的问题

已确认的约束与决策：
- 不修改 CRD Schema 和现有公共接口
- 优先复用现有错误处理与重试机制
- 方案及确认记录：<文档路径或讨论记录>

代码位置：
- 分支与提交：<分支名、提交 ID>
- 未提交变更：<是否存在；存在时写明位置和范围>

验证记录：
- 已执行：<实际命令、结果、日志位置、对应代码版本>
- 失败项：<错误摘要；没有则注明无>
- 尚未验证：<剩余验收项，例如 API 恢复后的自动调谐>

下一步：<具体操作及预期检查结果>
需要人工决定：<阻塞项；没有则注明无>
停止条件：<权限不足、需要改变已确认方案等>

1
2
3
4
5
6
7
8
9

+--------------------------------------------------------------------------------------------------------+
| npu-smi 22.0.4 Version: 22.0.4 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 7 310P3 | OK | NA 28 0 / 0 |
| 0 0 | 0000:00:07.0 | 0 1805 / 21527 |
+===============================+=================+======================================================+

1
2
3

helm repo add hami-charts https://project-hami.github.io/HAMi/
helm repo update
helm search repo hami-charts/hami --versions | head

1
2
3
4

helm install hami hami-charts/hami \
 --version 2.10.0 \
 -n kube-system \
 --set devices.ascend.enabled=true

1
2
3

root@lixd-npu-test2:~# kubectl -n kube-system get pod -l app.kubernetes.io/component=hami-scheduler
NAME READY STATUS RESTARTS AGE
hami-scheduler-7f554bd479-rsrb9 2/2 Running 0 3m33s

1

kubectl label node lixd-npu-test2 ascend=on --overwrite

1

kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/tags/v1.4.1/ascend-runtimeclass.yaml

1

kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/tags/v1.4.1/ascend-device-node-configmap.yaml

1

kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/tags/v1.4.1/ascend-device-plugin.yaml

1
2
3

kubectl -n kube-system set image \
 daemonset/hami-ascend-device-plugin \
 device-plugin=projecthami/ascend-device-plugin:v1.4.1

1
2
3

root@lixd-npu-test2:~# kubectl -n kube-system get pod -l app.kubernetes.io/component=hami-ascend-device-plugin
NAME READY STATUS RESTARTS AGE
hami-ascend-device-plugin-2z52v 1/1 Running 0 8m26s

1

kubectl describe node lixd-npu-test2 | grep "Capacity:" -A 7

1
2
3
4
5
6
7
8

Capacity:
 cpu: 16
 ephemeral-storage: 100676120576
 huawei.com/Ascend310P: 7
 hugepages-1Gi: 0
 hugepages-2Mi: 0
 memory: 65866796Ki
 pods: 110

1
2
3
4

resources:
 limits:
 huawei.com/Ascend310P: "1"
 huawei.com/Ascend310P-memory: "1024"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

root@lixd-npu-test2:~# npu-smi info -t template-info -i 7
+------------------------------------------------------------------------------------------+
|NPU instance template info is: |
|Name AICORE Memory AICPU VPC VENC JPEGD |
| GB PNGD VDEC JPEGE |
|==========================================================================================|
|vir01 1 3 1 1 0 2 |
| 0 1 1 |
+------------------------------------------------------------------------------------------+
|vir02 2 6 2 3 1 4 |
| 0 3 2 |
+------------------------------------------------------------------------------------------+
|vir02_1c 2 6 1 3 0 4 |
| 0 3 2 |
+------------------------------------------------------------------------------------------+
|vir04 4 12 4 6 2 8 |
| 0 6 4 |
+------------------------------------------------------------------------------------------+
|vir04_3c 4 12 3 6 1 8 |
| 0 6 4 |
+------------------------------------------------------------------------------------------+
|vir04_3c_ndvpp 4 12 3 0 0 0 |
| 0 0 0 |
+------------------------------------------------------------------------------------------+
|vir04_4c_dvpp 4 12 4 12 3 16 |
| 0 12 8 |
+------------------------------------------------------------------------------------------+

1
2
3

kubectl -n kube-system get cm hami-scheduler-device \
 -o jsonpath='{.data.device-config\.yaml}' \
 | grep -A24 'chipName: 310P3'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

- chipName: 310P3
 commonWord: Ascend310P
 resourceName: huawei.com/Ascend310P
 resourceMemoryName: huawei.com/Ascend310P-memory
 resourceCoreName: huawei.com/Ascend310P-core
 memoryAllocatable: 21527
 memoryCapacity: 24576
 aiCore: 8
 aiCPU: 7
 runtimeClassName: ascend
 templates:
 - name: vir01
 memory: 3072
 aiCore: 1
 aiCPU: 1
 - name: vir02
 memory: 6144
 aiCore: 2
 aiCPU: 2
 - name: vir04
 memory: 12288
 aiCore: 4
 aiCPU: 4

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

apiVersion: v1
kind: Pod
metadata:
 name: auto-1024
spec:
 schedulerName: hami-scheduler
 runtimeClassName: ascend
 restartPolicy: Never
 containers:
 - name: npu-test
 image: docker.io/ascendai/cann:7.0.1-310p-openeuler20.03-py3.8
 imagePullPolicy: IfNotPresent
 command: ["bash", "-lc"]
 args:
 - |
 echo "ASCEND_VISIBLE_DEVICES=$ASCEND_VISIBLE_DEVICES"
 echo "ASCEND_VNPU_SPECS=$ASCEND_VNPU_SPECS"
 npu-smi info
 sleep 3600
 resources:
 limits:
 huawei.com/Ascend310P: "1"
 huawei.com/Ascend310P-memory: "1024"

1

kubectl get pod auto-1024 -o wide

1
2

NAME READY STATUS RESTARTS AGE IP NODE
auto-1024 1/1 Running 0 21s 172.25.49.46 lixd-npu-test2

1
2

kubectl get pod auto-1024 \
 -o jsonpath='{.metadata.annotations.hami\.io/Ascend310P-devices-allocated}{"\n"}{.metadata.annotations.huawei\.com/Ascend310P}{"\n"}'

1
2

E0766E64-20C0E5F1-27941064-AED8030A-F6003019,Ascend310P,3072,0:;
[{"UUID":"E0766E64-20C0E5F1-27941064-AED8030A-F6003019","temp":"vir01","memory":3072}]

1
2
3
4
5

kubectl exec auto-1024 -- bash -lc '
 echo "ASCEND_VISIBLE_DEVICES=$ASCEND_VISIBLE_DEVICES"
 echo "ASCEND_VNPU_SPECS=$ASCEND_VNPU_SPECS"
 npu-smi info
'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

ASCEND_VISIBLE_DEVICES=0
ASCEND_VNPU_SPECS=vir01
+--------------------------------------------------------------------------------------------------------+
| npu-smi 22.0.4 Version: 22.0.4 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 7 310Pvir01 | OK | NA 28 0 / 0 |
| 0 0 | 0000:00:07.0 | 0 249 / 2690 |
+===============================+=================+======================================================+

1
2
3
4

NAME READY STATUS NODE
auto-1024 1/1 Running lixd-npu-test2
auto-4096 1/1 Running lixd-npu-test2
auto-7000 1/1 Running lixd-npu-test2

1
2
3
4
5
6

ASCEND_VISIBLE_DEVICES=0
ASCEND_VNPU_SPECS=vir02
+===============================+=================+======================================================+
| 7 310Pvir02 | OK | NA 28 0 / 0 |
| 0 0 | 0000:00:07.0 | 0 497 / 5381 |
+===============================+=================+======================================================+

1
2
3
4
5
6

ASCEND_VISIBLE_DEVICES=0
ASCEND_VNPU_SPECS=vir04
+===============================+=================+======================================================+
| 7 310Pvir04 | OK | NA 29 0 / 0 |
| 0 0 | 0000:00:07.0 | 0 995 / 10763 |
+===============================+=================+======================================================+

1
2
3
4
5
6
7
8

| Total number of vnpu: 3 |
+-------------------------------------------------------------------------------+
| Vnpu ID | Vgroup ID | Container ID | Status | Template Name |
+-------------------------------------------------------------------------------+
| 100 | 0 | ffffffffffff | 1 | vir01 |
| 101 | 1 | ffffffffffff | 1 | vir02 |
| 102 | 2 | ffffffffffff | 1 | vir04 |
+-------------------------------------------------------------------------------+

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

apiVersion: v1
kind: Pod
metadata:
 name: over-max-memory-idle-card
spec:
 schedulerName: hami-scheduler
 runtimeClassName: ascend
 restartPolicy: Never
 containers:
 - name: npu-test
 image: docker.io/ascendai/cann:7.0.1-310p-openeuler20.03-py3.8
 command: ["bash", "-lc"]
 args:
 - env | grep -E 'ASCEND|NPU' | sort; npu-smi info; sleep 300
 resources:
 limits:
 huawei.com/Ascend310P: "1"
 huawei.com/Ascend310P-memory: "13000"

1
2
3
4
5

pod/over-max-memory-idle-card created
pod/over-max-memory-idle-card condition met

NAME READY STATUS RESTARTS AGE IP NODE
over-max-memory-idle-card 1/1 Running 0 3m15s 172.25.49.59 lixd-npu-test2

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

{
 "allocated": "E0766E64-20C0E5F1-27941064-AED8030A-F6003019,Ascend310P,21527,0:;",
 "device": "[{\"UUID\":\"E0766E64-20C0E5F1-27941064-AED8030A-F6003019\",\"memory\":21527}]",
 "resources": {
 "limits": {
 "huawei.com/Ascend310P": "1",
 "huawei.com/Ascend310P-memory": "21527"
 },
 "requests": {
 "huawei.com/Ascend310P": "1",
 "huawei.com/Ascend310P-memory": "21527"
 }
 }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

ASCEND_AICPU_PATH=/usr/local/Ascend/ascend-toolkit/latest
ASCEND_DOCKER_RUNTIME=True
ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest
ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp
ASCEND_TOOLKIT_HOME=/usr/local/Ascend/ascend-toolkit/latest
ASCEND_VISIBLE_DEVICES=0
+--------------------------------------------------------------------------------------------------------+
| npu-smi 22.0.4 Version: 22.0.4 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 7 310P3 | OK | NA 28 0 / 0 |
| 0 0 | 0000:00:07.0 | 0 1805 / 21527 |
+===============================+=================+======================================================+

1
2

NAME READY UP-TO-DATE AVAILABLE
hami-310p-oversubscribe 7/8 8 7

1
2

Warning FailedScheduling hami-scheduler
0/1 nodes are available: 1 1/1 CardTimeSlicingExhausted.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

| Total number of vnpu: 7 |
+-------------------------------------------------------------------------------+
| 100 | 0 | ffffffffffff | 1 | vir01 |
| 101 | 0 | ffffffffffff | 1 | vir01 |
| 102 | 1 | ffffffffffff | 1 | vir01 |
| 103 | 1 | ffffffffffff | 1 | vir01 |
| 104 | 2 | ffffffffffff | 1 | vir01 |
| 105 | 2 | ffffffffffff | 1 | vir01 |
| 106 | 3 | ffffffffffff | 1 | vir01 |
+-------------------------------------------------------------------------------+

1
2
3

resources:
 limits:
 huawei.com/Ascend310P: "1"

1

huawei.com/Ascend310P-memory: "21527"

1

0/1 nodes are available: 1 1/1 CardInsufficientMemory.

1
2
3
4
5
6
7
8

NAME READY STATUS
auto-1024 1/1 Running
auto-4096 1/1 Running
auto-7000 1/1 Running
whole-after-slices 0/1 Pending

Warning FailedScheduling hami-scheduler
0/1 nodes are available: 1 1/1 CardInsufficientMemory.

1
2

failed to create containerd task: ... /usr/local/Ascend/Ascend-Docker-Runtime/ascend-docker-runtime
did not terminate successfully: exit status 1

1
2
3
4
5
6

| Total number of vnpu: 4 |
| Vnpu ID | Vgroup ID | Container ID | Status | Template Name |
| 100 | 0 | 000000000000 | 0 | vir01 |
| 101 | 0 | 000000000000 | 0 | vir01 |
| 102 | 2 | 000000000000 | 0 | vir04 | <- 残留
| 103 | 1 | 000000000000 | 0 | vir01 |

1
2

kubectl -n kube-system rollout restart daemonset/hami-ascend-device-plugin
npu-smi set -t destroy-vnpu -i 7 -c 0 -v VNPU_ID

1
2

| 7 310Pvir04 | OK | NA 33 0 / 0 |
| 0 0 | 0000:00:07.0 | 0 995 / 10763 |

1
2
3
4

+===============================+=================+======================================================+
| 7 310Pvir04 | OK | NA 33 3092 / 3092 |
| 0 0 | 0000:00:07.0 | 0 10763/ 10763 |
+===============================+=================+======================================================+

1
2

| 7 310Pvir04 | OK | NA 33 0 / 0 |
| 0 0 | 0000:00:07.0 | 0 995 / 10763 |

1
2
3
4

[100%] Building CXX object ... auto_gen_add_custom.cpp.o
[100%] Building CXX object ... auto_gen_matmul_custom.cpp.o
/usr/local/Ascend/ascend-toolkit/latest/compiler/ccec_compiler/bin/ld.lld -m aicorelinux ...
[100%] Built target main

1
2

output of add_custom:
8.000000 8.000000 8.000000 8.000000 ...

1
2

output of matmul:
8192.000000 8192.000000 8192.000000 8192.000000 ...

1
2
3
4
5

20:51:39.865
| 7 310P3 | OK | NA 34 20 / 20 |

20:51:41.424
| 7 310P3 | OK | NA 34 0 / 0 |

1
2

curl -s http://NODE_IP:31993/metrics \
 | grep -E 'hami_(gpu_shared_count|vgpu_memory_allocated_bytes|resource_quota_used)'

1
2
3
4
5
6
7
8
9

hami_gpu_shared_count{
 device_type="Ascend310P",
 node="lixd-npu-test2"
} 3

hami_resource_quota_used{
 namespace="hami-hard-slice-test",
 quota_name="huawei.com/Ascend310P-memory"
} 21504

1
2
3

hami_vgpu_memory_allocated_bytes{pod="auto-1024",namespace="hami-hard-slice-test"} 3.221225472e+09
hami_vgpu_memory_allocated_bytes{pod="auto-4096",namespace="hami-hard-slice-test"} 6.442450944e+09
hami_vgpu_memory_allocated_bytes{pod="auto-7000",namespace="hami-hard-slice-test"} 1.2884901888e+10

1
2

npu-smi info -l
npu-smi info -m

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

	Total Count : 1

	NPU ID : 13
	Product Name : IT21PDDA01
	Serial Number : 2106030635ZENC0xxx94
	Chip Count : 1

	NPU ID Chip ID Chip Logic ID Chip Name
	13 0 0 Ascend 310P3
	13 1 - Mcu

1
2
3
4

NPU ID: 13
Chip ID: 0
Chip Name: Ascend 310P3
Product Name: Atlas 300I Pro

1

npu-smi info -t product -i 13

1

	Product Type : Atlas 300I Pro

1

npu-smi info -t template-info -i 13

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

+------------------------------------------------------------------------------------------+
|NPU instance template info is: |
|Name AICORE Memory AICPU VPC VENC JPEGD |
| GB PNGD VDEC JPEGE |
|==========================================================================================|
|vir01 1 3 1 1 0 2 |
| 0 1 1 |
+------------------------------------------------------------------------------------------+
|vir02 2 6 2 3 1 4 |
| 0 3 2 |
+------------------------------------------------------------------------------------------+
|vir02_1c 2 6 1 3 0 4 |
| 0 3 2 |
+------------------------------------------------------------------------------------------+
|vir04 4 12 4 6 2 8 |
| 0 6 4 |
+------------------------------------------------------------------------------------------+
|vir04_3c 4 12 3 6 1 8 |
| 0 6 4 |
+------------------------------------------------------------------------------------------+
|vir04_3c_ndvpp 4 12 3 0 0 0 |
| 0 0 0 |
+------------------------------------------------------------------------------------------+
|vir04_4c_dvpp 4 12 4 12 3 16 |
| 0 12 8 |
+------------------------------------------------------------------------------------------+

1
2

dnf install -y docker-engine
systemctl enable --now docker

1
2

docker version
docker info --format 'server={{.ServerVersion}} arch={{.Architecture}} default-runtime={{.DefaultRuntime}}'

1

server=18.09.0 arch=x86_64 default-runtime=runc

1
2

npu-smi set -t create-vnpu -i 13 -c 0 -f vir01
npu-smi info -t info-vnpu -i 13 -c 0

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

	Status : OK
	Message : Create vnpu success

+-------------------------------------------------------------------------------+
| NPU resource static info as follow: |
| Format:Free/Total NA: Currently, query is not supported. |
| AICORE Memory AICPU VPC VENC VDEC JPEGD JPEGE PNGD |
| GB |
|===============================================================================|
| 7/8 18/21 6/7 11/12 3/3 11/12 14/16 7/8 NA/NA |
+-------------------------------------------------------------------------------+
| Total number of vnpu: 1 |
+-------------------------------------------------------------------------------+
| Vnpu ID | Vgroup ID | Container ID | Status | Template Name |
+-------------------------------------------------------------------------------+
| 100 | 0 | 000000000000 | 0 | vir01 |
+-------------------------------------------------------------------------------+

1
2
3
4

ls -l /dev/vdavinci100 \
 /dev/davinci_manager \
 /dev/devmm_svm \
 /dev/hisi_hdc

1
2
3
4

crw-rw---- 1 HwHiAiUser HwHiAiUser 239, 0 Sep 17 17:42 /dev/davinci_manager
crw-rw---- 1 HwHiAiUser HwHiAiUser 234, 0 Sep 17 17:42 /dev/devmm_svm
crw-rw---- 1 HwHiAiUser HwHiAiUser 511, 0 Sep 17 17:42 /dev/hisi_hdc
crw------- 1 HwHiAiUser HwHiAiUser 236, 100 Sep 17 20:49 /dev/vdavinci100

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

IMG=ascendai/cann:7.0.1-310p-openeuler20.03-py3.8

docker run --rm \
 --device=/dev/vdavinci100:/dev/davinci100 \
 --device=/dev/davinci_manager \
 --device=/dev/devmm_svm \
 --device=/dev/hisi_hdc \
 -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi:ro \
 -v /usr/local/Ascend:/usr/local/Ascend:ro \
 -e LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/driver/lib64/driver:/usr/local/Ascend/driver/lib64 \
 $IMG \
 npu-smi info

1
2
3
4
5
6
7
8
9

+--------------------------------------------------------------------------------------------------------+
| npu-smi 22.0.4 Version: 22.0.4 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 13 310Pvir01 | OK | NA 56 0 / 0 |
| 0 0 | 0000:00:0D.0 | 0 312 / 2690 |
+===============================+=================+======================================================+

1
2
3
4
5
6
7
8
9

mkdir -p /tmp/ascend-docker-runtime
cd /tmp/ascend-docker-runtime

wget -O Ascend-docker-runtime_6.0.0_linux-x86_64.run \
 https://gitee.com/ascend/mind-cluster/releases/download/v6.0.0/Ascend-docker-runtime_6.0.0_linux-x86_64.run

sha256sum Ascend-docker-runtime_6.0.0_linux-x86_64.run
chmod u+x Ascend-docker-runtime_6.0.0_linux-x86_64.run
./Ascend-docker-runtime_6.0.0_linux-x86_64.run --install

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

2e3800050d4203daa87937748a436ba0c32fabbc8693ac597ee40ba5b6f7c6af Ascend-docker-runtime_6.0.0_linux-x86_64.run
Uncompressing ascend-docker-runtime 0% 57% 100%
[INFO] installing ascend docker runtime
[INFO] platform(x86_64) matched!
[INFO] install executable files success
[INFO] install scene is 'docker'.
[INFO] /etc/docker/daemon.json modify success
[INFO] Ascend Docker Runtime has been installed in: /usr/local/Ascend/Ascend-Docker-Runtime
[INFO] The version of Ascend Docker Runtime is: 6.0.0
[INFO] please reboot daemon and container engine to take effect
[INFO] Ascend Docker Runtime install success

1
2
3

systemctl daemon-reload
systemctl restart docker
docker info --format 'server={{.ServerVersion}} arch={{.Architecture}} default-runtime={{.DefaultRuntime}}'

1

server=18.09.0 arch=x86_64 default-runtime=ascend

1
2
3
4
5
6
7
8
9

{
 "default-runtime": "ascend",
 "runtimes": {
 "ascend": {
 "path": "/usr/local/Ascend/Ascend-Docker-Runtime/ascend-docker-runtime",
 "runtimeArgs": []
 }
 }
}

1
2
3
4
5
6
7

IMG=ascendai/cann:7.0.1-310p-openeuler20.03-py3.8

docker run --rm --runtime=ascend \
 -e ASCEND_VISIBLE_DEVICES=100 \
 -e ASCEND_RUNTIME_OPTIONS=VIRTUAL \
 $IMG \
 npu-smi info

1
2
3
4
5
6
7
8
9

+--------------------------------------------------------------------------------------------------------+
| npu-smi 22.0.4 Version: 22.0.4 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 13 310Pvir01 | OK | NA 56 0 / 0 |
| 0 0 | 0000:00:0D.0 | 0 312 / 2690 |
+===============================+=================+======================================================+

1
2

npu-smi set -t destroy-vnpu -i 13 -c 0 -v 100
npu-smi info -t info-vnpu -i 13 -c 0

1

npu-smi set -t vnpu-cfg-recover -d 0

1
2

	Status : OK
	Message : The VNPU config recover mode Disable is set successfully.

1
2
3
4
5

docker run --rm --runtime=ascend \
 -e ASCEND_VISIBLE_DEVICES=0 \
 -e ASCEND_VNPU_SPECS=vir01 \
 $IMG \
 npu-smi info

1
2
3
4
5
6
7
8
9

+--------------------------------------------------------------------------------------------------------+
| npu-smi 22.0.4 Version: 22.0.4 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 13 310Pvir01 | OK | NA 56 0 / 0 |
| 0 0 | 0000:00:0D.0 | 0 312 / 2690 |
+===============================+=================+======================================================+

1

npu-smi info -t info-vnpu -i 13 -c 0

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

+-------------------------------------------------------------------------------+
| NPU resource static info as follow: |
| Format:Free/Total NA: Currently, query is not supported. |
| AICORE Memory AICPU VPC VENC VDEC JPEGD JPEGE PNGD |
| GB |
+===============================================================================+
| 8/8 21/21 7/7 12/12 3/3 12/12 16/16 8/8 NA/NA |
+-------------------------------------------------------------------------------+
| Total number of vnpu: 0 |
+-------------------------------------------------------------------------------+
| Vnpu ID | Vgroup ID | Container ID | Status | Template Name |
+-------------------------------------------------------------------------------+