k3s动态扩容

config.yml

@@ -46,7 +46,7 @@ redis:
 #  address: 116.204.74.41:8500
 k8s:
   apiServer: "https://192.168.3.37:6443"
-  token: "eyJhbGciOiJSUzI1NiIsImtpZCI6IlR6X0QtelVDYkZPYnpjTDlfamZzOEFjbERJb2dUMTRqMjNfYUNncGcwRW8ifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiLCJrM3MiXSwiZXhwIjoxNzc2MzA1MDUyLCJpYXQiOjE3NzYzMDE0NTIsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwianRpIjoiMjAwYWRlM2YtMTZhNC00ZGVlLTgzOWItZWY2MzgwYjNlNWI5Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJkYXNoYm9hcmQtYWRtaW4iLCJ1aWQiOiIwMTQ0N2IwOS1hNzA5LTQ1YWUtYWRjOS1kNzE2NjQzNGNmM2UifX0sIm5iZiI6MTc3NjMwMTQ1Miwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmVybmV0ZXMtZGFzaGJvYXJkOmRhc2hib2FyZC1hZG1pbiJ9.Wn6PQuIy_SFG40ZFZ1w7yK7mk40CuRr_Ybv5Vjm2nXmljEEbS8SgEtivFkCcOd1jT-KJ2miG-d0rNqhur5EsJSo28Nzwr71gmWl-ZGyJVeyEMfPV5WshE-_BEP_P63Ne4oWf35MlxrMRgJmFxcSED4PyCUXa2V0qylOWgcQ_7UTgZTR4KNfwnFS7LHRwFF2zsPyNsA4tQtJmJnUUgJcNLCiVw6p3TRe9xYp6f7rk9Aoyin_Gg_52fmWZe9K4L_7TlF75NoiiS_dw563Vf0zWeaQ-Nchy0OstJKhUrNAFOxJJY5n1NRJXeWt9XwZ43nkjbu9JKl6ra3bcNip-go7nXQ"
+  token: "eyJhbGciOiJSUzI1NiIsImtpZCI6IlR6X0QtelVDYkZPYnpjTDlfamZzOEFjbERJb2dUMTRqMjNfYUNncGcwRW8ifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiLCJrM3MiXSwiZXhwIjoxODA3ODY3MTA1LCJpYXQiOjE3NzYzMzExMDUsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwianRpIjoiNjAwMGY0ODctZTQ4Ni00NTUxLWIyNjgtMzE1MWE1MDE5YjU4Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJkYXNoYm9hcmQtYWRtaW4iLCJ1aWQiOiJlZjAxY2IxNS0xNTc0LTRlYTYtODg4Ny03YjZhODY3OWFkYmIifX0sIm5iZiI6MTc3NjMzMTEwNSwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmVybmV0ZXMtZGFzaGJvYXJkOmRhc2hib2FyZC1hZG1pbiJ9.Z5MEy-dhWq6lruuR6QSI--cYRZzDoeEes78a4lLMtXa8lSLt1FZy16kNVn2pJli74RSk0kKS2F5GAatyuyGjF_yB7Tm1Tb7iCjzeUM7xRpdvXEmk4MUImlzwGeW31NXpwFrfR0oPS-9TLlbahEmsgwL0DUOC0CekSiToVbIduwEmmrB0FMPFayr5_wqGDmxdqtaAH3K-LJNCqPiAMqevsxTebrhyJSrThU-Pi7iJYm_sUd8VHKrRxrwDiGA77j4lfita_hZSdyrZe8qsrbGqomHxFWk9ZzcJJ9q7OLS13OXo-Xbcu-_TZcfDOreZhune9ctM2lmuOtmYnad47gYRcA"
   kubeconfig: ""
   namespace: "default"
 #    pass: jiahui8888

deploy-k3s.ps1

@@ -48,11 +48,78 @@ if ($LASTEXITCODE -ne 0) {
 
 Write-Host "[OK] Image imported" -ForegroundColor Green
 
+Write-Host "`n[3.5/7] Check and fix metrics-server..." -ForegroundColor Yellow
+$metricsPod = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath='{.items[0].metadata.name}' 2>$null
+if ($metricsPod) {
+    $podStatus = kubectl -n kube-system get pod $metricsPod -o jsonpath='{.status.phase}' 2>$null
+    if ($podStatus -ne "Running") {
+        Write-Host "  metrics-server status: $podStatus, attempting to fix..." -ForegroundColor Yellow
+
+        $podDetail = kubectl -n kube-system describe pod $metricsPod 2>&1
+        if ($podDetail -match "ImagePullBackOff|ErrImagePull") {
+            Write-Host "  Detected image pull issue, using domestic registry..." -ForegroundColor Cyan
+
+            try {
+                docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.3 2>$null
+                if ($LASTEXITCODE -eq 0) {
+                    docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.3 rancher/mirrored-metrics-server:v0.8.0
+                    $msTarPath = "$PWD\metrics-server.tar"
+                    docker save -o $msTarPath rancher/mirrored-metrics-server:v0.8.0
+                    if ($LASTEXITCODE -eq 0) {
+                        docker cp $msTarPath "${k3sContainer}:/tmp/metrics-server.tar"
+                        docker exec $k3sContainer ctr -n k8s.io images import /tmp/metrics-server.tar
+                        docker exec $k3sContainer rm /tmp/metrics-server.tar
+                        Remove-Item $msTarPath -ErrorAction SilentlyContinue
+
+                        kubectl -n kube-system delete pod $metricsPod --force --grace-period=0
+                        Write-Host "  [OK] metrics-server image imported and pod restarted" -ForegroundColor Green
+                    }
+                }
+            } catch {
+                Write-Host "  [WARN] Failed to import metrics-server image" -ForegroundColor Yellow
+            }
+        } else {
+            kubectl -n kube-system delete pod $metricsPod
+            Write-Host "  [OK] metrics-server pod restarted" -ForegroundColor Green
+        }
+    } else {
+        Write-Host "  [OK] metrics-server is running" -ForegroundColor Green
+    }
+} else {
+    Write-Host "  metrics-server not found, deploying..." -ForegroundColor Yellow
+    kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml 2>$null
+}
+
+Write-Host "  Waiting for metrics-server to be ready..." -ForegroundColor Cyan
+$retryCount = 0
+$maxRetries = 40
+while ($retryCount -lt $maxRetries) {
+    $msPod = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath='{.items[0].status.phase}' 2>$null
+    $msReady = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath="{.items[0].status.conditions[?(@.type=='Ready')].status}" 2>$null
+
+    if ($msPod -eq "Running" -and $msReady -eq "True") {
+        Write-Host "  [OK] metrics-server is ready" -ForegroundColor Green
+        break
+    }
+
+    Start-Sleep -Seconds 5
+    $retryCount++
+    if ($retryCount % 8 -eq 0) {
+        Write-Host "  Still waiting... ($retryCount/$maxRetries) - Status: $msPod" -ForegroundColor Gray
+    }
+}
+
+if ($retryCount -eq $maxRetries) {
+    Write-Host "  [WARN] metrics-server may not be ready yet, HPA might not work immediately" -ForegroundColor Yellow
+    Write-Host "  Run '.\fix-metrics-server.ps1' manually if needed" -ForegroundColor Gray
+}
+
 Write-Host "`n[4/7] Deploy to K3s..." -ForegroundColor Yellow
 kubectl apply -f k8s/configmap.yaml
 kubectl apply -f k8s/pvc.yaml
 kubectl apply -f k8s/deployment.yaml
 kubectl apply -f k8s/service.yaml
+kubectl apply -f k8s/hpa.yaml
 Write-Host "[OK] Resources created" -ForegroundColor Green
 
 Write-Host "`n[5/7] Wait for Pod ready..." -ForegroundColor Yellow
@@ -66,11 +133,25 @@ kubectl get svc data-engine-service
 Write-Host "`n[7/7] Logs:" -ForegroundColor Yellow
 kubectl logs -l app=data-engine --tail=50
 
+Write-Host "`nHPA Status:" -ForegroundColor Cyan
+try {
+    $hpaExists = kubectl get hpa data-engine-hpa 2>&1
+    if ($LASTEXITCODE -eq 0) {
+        Write-Host $hpaExists -ForegroundColor Green
+        kubectl describe hpa data-engine-hpa | Select-String "Metrics|Conditions" -Context 0,2
+    } else {
+        Write-Host "[WARN] HPA not available yet, please check manually later" -ForegroundColor Yellow
+    }
+} catch {
+    Write-Host "[WARN] HPA status check failed, please check manually later" -ForegroundColor Yellow
+}
+
 Write-Host "`n=== Deployment Success ===" -ForegroundColor Green
 Write-Host "Access: http://<node-ip>:30301" -ForegroundColor Cyan
 Write-Host "`nCommands:" -ForegroundColor Yellow
 Write-Host "  Logs:   kubectl logs -l app=data-engine -f" -ForegroundColor White
 Write-Host "  Status: kubectl get pods -l app=data-engine" -ForegroundColor White
+Write-Host "  HPA:    kubectl get hpa data-engine-hpa -w" -ForegroundColor White
 Write-Host "  Delete: kubectl delete -f k8s/" -ForegroundColor White
 
 Write-Host "`nCleanup..." -ForegroundColor Yellow

fix-metrics-server.ps1

@@ -0,0 +1,60 @@
+Write-Host "=== Fix metrics-server ===" -ForegroundColor Cyan
+Write-Host ""
+
+$k3sContainer = "k3s-server"
+$TARGET_IMAGE = "rancher/mirrored-metrics-server:v0.8.0"
+
+Write-Host "[Step 1] Pull image from domestic registry..." -ForegroundColor Yellow
+docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.3
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "[ERROR] Failed to pull image" -ForegroundColor Red
+    exit 1
+}
+
+Write-Host "[Step 2] Tag image to match K3s requirement..." -ForegroundColor Yellow
+docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.3 $TARGET_IMAGE
+Write-Host "[OK] Image tagged as $TARGET_IMAGE" -ForegroundColor Green
+
+Write-Host "[Step 3] Save to tar file..." -ForegroundColor Yellow
+$tarPath = "$PWD\metrics-server.tar"
+docker save -o $tarPath $TARGET_IMAGE
+Write-Host "[OK] Saved: $([math]::Round((Get-Item $tarPath).Length / 1MB, 2)) MB" -ForegroundColor Green
+
+Write-Host "[Step 4] Copy to K3s container..." -ForegroundColor Yellow
+docker cp $tarPath "${k3sContainer}:/tmp/metrics-server.tar"
+Write-Host "[OK] Copied" -ForegroundColor Green
+
+Write-Host "[Step 5] Import in K3s..." -ForegroundColor Yellow
+docker exec $k3sContainer ctr -n k8s.io images import /tmp/metrics-server.tar
+docker exec $k3sContainer rm /tmp/metrics-server.tar
+Remove-Item $tarPath
+Write-Host "[OK] Imported" -ForegroundColor Green
+
+Write-Host "[Step 6] Restart metrics-server pod..." -ForegroundColor Yellow
+kubectl -n kube-system delete pod -l k8s-app=metrics-server --force --grace-period=0
+
+Write-Host "Waiting for pod to start..." -ForegroundColor Cyan
+Start-Sleep -Seconds 15
+
+for ($i = 1; $i -le 30; $i++) {
+    $status = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath='{.items[0].status.phase}' 2>$null
+    $ready = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>$null
+
+    if ($status -eq "Running" -and $ready -eq "True") {
+        Write-Host "[SUCCESS] metrics-server is ready!" -ForegroundColor Green
+        break
+    }
+
+    if ($i % 5 -eq 0) {
+        Write-Host "  Waiting... ($i/30) Current: $status" -ForegroundColor Gray
+    }
+    Start-Sleep -Seconds 5
+}
+
+Write-Host ""
+Write-Host "Testing metrics API..." -ForegroundColor Cyan
+Start-Sleep -Seconds 5
+kubectl top pods
+
+Write-Host ""
+Write-Host "=== Done ===" -ForegroundColor Green

k8s/hpa.yaml

@@ -0,0 +1,69 @@
+# Horizontal Pod Autoscaler - 水平Pod自动扩缩容配置
+# 根据 CPU 和内存使用率自动调整 Pod 副本数量
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: data-engine-hpa  # HPA 资源名称
+  namespace: default     # 命名空间
+spec:
+  # 指定要自动扩缩容的目标 Deployment
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: data-engine
+  
+  # 副本数量范围
+  minReplicas: 1   # 最小副本数：始终保持至少 1 个 Pod 运行
+  maxReplicas: 10  # 最大副本数：防止无限扩容，最多扩展到 10 个 Pod
+  
+  # 扩缩容触发指标（满足任一条件即触发）
+  metrics:
+    # CPU 使用率指标
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 70  # 当所有 Pod 的平均 CPU 使用率超过 70% 时触发扩容
+    
+    # 内存使用率指标
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: 80  # 当所有 Pod 的平均内存使用率超过 80% 时触发扩容
+  
+  # 扩缩容行为策略
+  behavior:
+    # 向上扩容策略（快速响应负载增加）
+    scaleUp:
+      stabilizationWindowSeconds: 60  # 稳定窗口期：60 秒内只基于最高需求决策，防止抖动
+      policies:
+        # 策略1：按固定数量扩容
+        - type: Pods
+          value: 2              # 每次最多增加 2 个 Pod
+          periodSeconds: 60     # 每 60 秒评估一次
+        
+        # 策略2：按比例扩容
+        - type: Percent
+          value: 50             # 每次最多增加当前副本数的 50%
+          periodSeconds: 60     # 每 60 秒评估一次
+      
+      selectPolicy: Max  # 选择两个策略中扩容幅度更大的方式（更激进）
+    
+    # 向下缩容策略（保守缩容，避免频繁波动）
+    scaleDown:
+      stabilizationWindowSeconds: 300  # 稳定窗口期：300 秒（5分钟），比扩容更保守
+      policies:
+        # 策略1：按固定数量缩容
+        - type: Pods
+          value: 1              # 每次最多减少 1 个 Pod
+          periodSeconds: 60     # 每 60 秒评估一次
+        
+        # 策略2：按比例缩容
+        - type: Percent
+          value: 25             # 每次最多减少当前副本数的 25%
+          periodSeconds: 60     # 每 60 秒评估一次
+      
+      selectPolicy: Min  # 选择两个策略中缩容幅度更小的方式（更保守）

test-hpa-strong.ps1

@@ -0,0 +1,133 @@
+Write-Host "=== HPA Stress Test ===" -ForegroundColor Cyan
+Write-Host ""
+
+# 配置参数
+$url = "http://localhost:30301"
+$jobCount = 50
+$requestPerJob = 5000
+
+Write-Host "Configuration:" -ForegroundColor Yellow
+Write-Host "  Target URL: $url" -ForegroundColor White
+Write-Host "  Concurrent Jobs: $jobCount" -ForegroundColor White
+Write-Host "  Requests per Job: $requestPerJob" -ForegroundColor White
+Write-Host "  Total Requests: $($jobCount * $requestPerJob)" -ForegroundColor White
+Write-Host ""
+
+# 清理旧任务
+Write-Host "[1/4] Cleaning up old jobs..." -ForegroundColor Cyan
+Get-Job | Where-Object { $_.Name -like "HPA-*" } | Stop-Job | Remove-Job
+Start-Sleep -Seconds 2
+Write-Host "[OK] Cleaned" -ForegroundColor Green
+Write-Host ""
+
+# 启动并发任务
+Write-Host "[2/4] Starting $jobCount concurrent jobs..." -ForegroundColor Cyan
+$jobs = @()
+
+for ($i = 1; $i -le $jobCount; $i++) {
+    try {
+        $job = Start-Job -Name "HPA-$i" -ScriptBlock {
+            param($targetUrl, $totalRequests)
+            $success = 0
+            $failed = 0
+
+            for ($j = 1; $j -le $totalRequests; $j++) {
+                try {
+                    $response = Invoke-WebRequest -Uri $targetUrl -Method GET -TimeoutSec 3 -UseBasicParsing
+                    if ($response.StatusCode -eq 200) {
+                        $success++
+                    }
+                } catch {
+                    $failed++
+                }
+
+                # 每 500 个请求短暂休息，避免完全耗尽资源
+                if ($j % 500 -eq 0) {
+                    Start-Sleep -Milliseconds 50
+                }
+            }
+
+            return @{
+                Success = $success
+                Failed = $failed
+            }
+        } -ArgumentList $url, $requestPerJob
+
+        $jobs += $job
+
+        if ($i % 10 -eq 0) {
+            Write-Host "  Started $i/$jobCount jobs..." -ForegroundColor Gray
+        }
+    } catch {
+        Write-Host "  Failed to start job $i : $_" -ForegroundColor Red
+    }
+}
+
+Write-Host ""
+Write-Host "[OK] All $jobCount jobs started" -ForegroundColor Green
+Write-Host ""
+
+# 显示监控命令
+Write-Host "[3/4] Monitoring Commands (run in separate windows):" -ForegroundColor Cyan
+Write-Host "  Window 1 - HPA Status:" -ForegroundColor Yellow
+Write-Host "    kubectl get hpa data-engine-hpa -w" -ForegroundColor White
+Write-Host ""
+Write-Host "  Window 2 - Pod Status:" -ForegroundColor Yellow
+Write-Host "    kubectl get pods -l app=data-engine -w" -ForegroundColor White
+Write-Host ""
+Write-Host "  Window 3 - Resource Usage:" -ForegroundColor Yellow
+Write-Host "    while(`$true) { Clear-Host; kubectl top pods -l app=data-engine; kubectl get hpa data-engine-hpa; Start-Sleep 3 }" -ForegroundColor White
+Write-Host ""
+
+# 等待用户确认
+Write-Host "[4/4] Load test is running..." -ForegroundColor Green
+Write-Host "Press any key to stop all jobs and see results..." -ForegroundColor Yellow
+$null = $Host.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown")
+Write-Host ""
+
+# 停止所有任务
+Write-Host "Stopping all jobs..." -ForegroundColor Cyan
+Get-Job -Name "HPA-*" | Stop-Job
+
+# 收集结果
+Write-Host "Collecting results..." -ForegroundColor Cyan
+$results = Get-Job -Name "HPA-*" | Receive-Job -Wait
+Get-Job -Name "HPA-*" | Remove-Job
+
+# 统计
+$totalSuccess = 0
+$totalFailed = 0
+foreach ($result in $results) {
+    $totalSuccess += $result.Success
+    $totalFailed += $result.Failed
+}
+
+Write-Host ""
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host "         TEST RESULTS" -ForegroundColor Cyan
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host ""
+Write-Host "Total Requests:     $($totalSuccess + $totalFailed)" -ForegroundColor White
+Write-Host "Successful:         $totalSuccess" -ForegroundColor Green
+Write-Host "Failed:             $totalFailed" -ForegroundColor $(if ($totalFailed -gt 0) { "Red" } else { "Green" })
+Write-Host ""
+
+# 显示最终状态
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host "         FINAL STATUS" -ForegroundColor Cyan
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host ""
+Write-Host "[HPA Status]" -ForegroundColor Yellow
+kubectl get hpa data-engine-hpa
+Write-Host ""
+Write-Host "[Pod List]" -ForegroundColor Yellow
+kubectl get pods -l app=data-engine
+Write-Host ""
+Write-Host "[Resource Usage]" -ForegroundColor Yellow
+kubectl top pods -l app=data-engine 2>$null
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "  Metrics not available" -ForegroundColor Gray
+}
+Write-Host ""
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host "Test completed!" -ForegroundColor Green