diff --git a/config.yml b/config.yml index 30188de..ed40704 100644 --- a/config.yml +++ b/config.yml @@ -46,7 +46,7 @@ redis: # address: 116.204.74.41:8500 k8s: apiServer: "https://192.168.3.37:6443" - token: "eyJhbGciOiJSUzI1NiIsImtpZCI6IlR6X0QtelVDYkZPYnpjTDlfamZzOEFjbERJb2dUMTRqMjNfYUNncGcwRW8ifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiLCJrM3MiXSwiZXhwIjoxNzc2MzA1MDUyLCJpYXQiOjE3NzYzMDE0NTIsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwianRpIjoiMjAwYWRlM2YtMTZhNC00ZGVlLTgzOWItZWY2MzgwYjNlNWI5Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJkYXNoYm9hcmQtYWRtaW4iLCJ1aWQiOiIwMTQ0N2IwOS1hNzA5LTQ1YWUtYWRjOS1kNzE2NjQzNGNmM2UifX0sIm5iZiI6MTc3NjMwMTQ1Miwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmVybmV0ZXMtZGFzaGJvYXJkOmRhc2hib2FyZC1hZG1pbiJ9.Wn6PQuIy_SFG40ZFZ1w7yK7mk40CuRr_Ybv5Vjm2nXmljEEbS8SgEtivFkCcOd1jT-KJ2miG-d0rNqhur5EsJSo28Nzwr71gmWl-ZGyJVeyEMfPV5WshE-_BEP_P63Ne4oWf35MlxrMRgJmFxcSED4PyCUXa2V0qylOWgcQ_7UTgZTR4KNfwnFS7LHRwFF2zsPyNsA4tQtJmJnUUgJcNLCiVw6p3TRe9xYp6f7rk9Aoyin_Gg_52fmWZe9K4L_7TlF75NoiiS_dw563Vf0zWeaQ-Nchy0OstJKhUrNAFOxJJY5n1NRJXeWt9XwZ43nkjbu9JKl6ra3bcNip-go7nXQ" + token: "eyJhbGciOiJSUzI1NiIsImtpZCI6IlR6X0QtelVDYkZPYnpjTDlfamZzOEFjbERJb2dUMTRqMjNfYUNncGcwRW8ifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiLCJrM3MiXSwiZXhwIjoxODA3ODY3MTA1LCJpYXQiOjE3NzYzMzExMDUsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwianRpIjoiNjAwMGY0ODctZTQ4Ni00NTUxLWIyNjgtMzE1MWE1MDE5YjU4Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJkYXNoYm9hcmQtYWRtaW4iLCJ1aWQiOiJlZjAxY2IxNS0xNTc0LTRlYTYtODg4Ny03YjZhODY3OWFkYmIifX0sIm5iZiI6MTc3NjMzMTEwNSwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmVybmV0ZXMtZGFzaGJvYXJkOmRhc2hib2FyZC1hZG1pbiJ9.Z5MEy-dhWq6lruuR6QSI--cYRZzDoeEes78a4lLMtXa8lSLt1FZy16kNVn2pJli74RSk0kKS2F5GAatyuyGjF_yB7Tm1Tb7iCjzeUM7xRpdvXEmk4MUImlzwGeW31NXpwFrfR0oPS-9TLlbahEmsgwL0DUOC0CekSiToVbIduwEmmrB0FMPFayr5_wqGDmxdqtaAH3K-LJNCqPiAMqevsxTebrhyJSrThU-Pi7iJYm_sUd8VHKrRxrwDiGA77j4lfita_hZSdyrZe8qsrbGqomHxFWk9ZzcJJ9q7OLS13OXo-Xbcu-_TZcfDOreZhune9ctM2lmuOtmYnad47gYRcA" kubeconfig: "" namespace: "default" # pass: jiahui8888 diff --git a/deploy-k3s.ps1 b/deploy-k3s.ps1 index 69c5e16..296bb66 100644 --- a/deploy-k3s.ps1 +++ b/deploy-k3s.ps1 @@ -48,11 +48,78 @@ if ($LASTEXITCODE -ne 0) { Write-Host "[OK] Image imported" -ForegroundColor Green +Write-Host "`n[3.5/7] Check and fix metrics-server..." -ForegroundColor Yellow +$metricsPod = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath='{.items[0].metadata.name}' 2>$null +if ($metricsPod) { + $podStatus = kubectl -n kube-system get pod $metricsPod -o jsonpath='{.status.phase}' 2>$null + if ($podStatus -ne "Running") { + Write-Host " metrics-server status: $podStatus, attempting to fix..." -ForegroundColor Yellow + + $podDetail = kubectl -n kube-system describe pod $metricsPod 2>&1 + if ($podDetail -match "ImagePullBackOff|ErrImagePull") { + Write-Host " Detected image pull issue, using domestic registry..." -ForegroundColor Cyan + + try { + docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.3 2>$null + if ($LASTEXITCODE -eq 0) { + docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.3 rancher/mirrored-metrics-server:v0.8.0 + $msTarPath = "$PWD\metrics-server.tar" + docker save -o $msTarPath rancher/mirrored-metrics-server:v0.8.0 + if ($LASTEXITCODE -eq 0) { + docker cp $msTarPath "${k3sContainer}:/tmp/metrics-server.tar" + docker exec $k3sContainer ctr -n k8s.io images import /tmp/metrics-server.tar + docker exec $k3sContainer rm /tmp/metrics-server.tar + Remove-Item $msTarPath -ErrorAction SilentlyContinue + + kubectl -n kube-system delete pod $metricsPod --force --grace-period=0 + Write-Host " [OK] metrics-server image imported and pod restarted" -ForegroundColor Green + } + } + } catch { + Write-Host " [WARN] Failed to import metrics-server image" -ForegroundColor Yellow + } + } else { + kubectl -n kube-system delete pod $metricsPod + Write-Host " [OK] metrics-server pod restarted" -ForegroundColor Green + } + } else { + Write-Host " [OK] metrics-server is running" -ForegroundColor Green + } +} else { + Write-Host " metrics-server not found, deploying..." -ForegroundColor Yellow + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml 2>$null +} + +Write-Host " Waiting for metrics-server to be ready..." -ForegroundColor Cyan +$retryCount = 0 +$maxRetries = 40 +while ($retryCount -lt $maxRetries) { + $msPod = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath='{.items[0].status.phase}' 2>$null + $msReady = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath="{.items[0].status.conditions[?(@.type=='Ready')].status}" 2>$null + + if ($msPod -eq "Running" -and $msReady -eq "True") { + Write-Host " [OK] metrics-server is ready" -ForegroundColor Green + break + } + + Start-Sleep -Seconds 5 + $retryCount++ + if ($retryCount % 8 -eq 0) { + Write-Host " Still waiting... ($retryCount/$maxRetries) - Status: $msPod" -ForegroundColor Gray + } +} + +if ($retryCount -eq $maxRetries) { + Write-Host " [WARN] metrics-server may not be ready yet, HPA might not work immediately" -ForegroundColor Yellow + Write-Host " Run '.\fix-metrics-server.ps1' manually if needed" -ForegroundColor Gray +} + Write-Host "`n[4/7] Deploy to K3s..." -ForegroundColor Yellow kubectl apply -f k8s/configmap.yaml kubectl apply -f k8s/pvc.yaml kubectl apply -f k8s/deployment.yaml kubectl apply -f k8s/service.yaml +kubectl apply -f k8s/hpa.yaml Write-Host "[OK] Resources created" -ForegroundColor Green Write-Host "`n[5/7] Wait for Pod ready..." -ForegroundColor Yellow @@ -66,11 +133,25 @@ kubectl get svc data-engine-service Write-Host "`n[7/7] Logs:" -ForegroundColor Yellow kubectl logs -l app=data-engine --tail=50 +Write-Host "`nHPA Status:" -ForegroundColor Cyan +try { + $hpaExists = kubectl get hpa data-engine-hpa 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host $hpaExists -ForegroundColor Green + kubectl describe hpa data-engine-hpa | Select-String "Metrics|Conditions" -Context 0,2 + } else { + Write-Host "[WARN] HPA not available yet, please check manually later" -ForegroundColor Yellow + } +} catch { + Write-Host "[WARN] HPA status check failed, please check manually later" -ForegroundColor Yellow +} + Write-Host "`n=== Deployment Success ===" -ForegroundColor Green Write-Host "Access: http://:30301" -ForegroundColor Cyan Write-Host "`nCommands:" -ForegroundColor Yellow Write-Host " Logs: kubectl logs -l app=data-engine -f" -ForegroundColor White Write-Host " Status: kubectl get pods -l app=data-engine" -ForegroundColor White +Write-Host " HPA: kubectl get hpa data-engine-hpa -w" -ForegroundColor White Write-Host " Delete: kubectl delete -f k8s/" -ForegroundColor White Write-Host "`nCleanup..." -ForegroundColor Yellow diff --git a/fix-metrics-server.ps1 b/fix-metrics-server.ps1 new file mode 100644 index 0000000..494392e --- /dev/null +++ b/fix-metrics-server.ps1 @@ -0,0 +1,60 @@ +Write-Host "=== Fix metrics-server ===" -ForegroundColor Cyan +Write-Host "" + +$k3sContainer = "k3s-server" +$TARGET_IMAGE = "rancher/mirrored-metrics-server:v0.8.0" + +Write-Host "[Step 1] Pull image from domestic registry..." -ForegroundColor Yellow +docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.3 +if ($LASTEXITCODE -ne 0) { + Write-Host "[ERROR] Failed to pull image" -ForegroundColor Red + exit 1 +} + +Write-Host "[Step 2] Tag image to match K3s requirement..." -ForegroundColor Yellow +docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.3 $TARGET_IMAGE +Write-Host "[OK] Image tagged as $TARGET_IMAGE" -ForegroundColor Green + +Write-Host "[Step 3] Save to tar file..." -ForegroundColor Yellow +$tarPath = "$PWD\metrics-server.tar" +docker save -o $tarPath $TARGET_IMAGE +Write-Host "[OK] Saved: $([math]::Round((Get-Item $tarPath).Length / 1MB, 2)) MB" -ForegroundColor Green + +Write-Host "[Step 4] Copy to K3s container..." -ForegroundColor Yellow +docker cp $tarPath "${k3sContainer}:/tmp/metrics-server.tar" +Write-Host "[OK] Copied" -ForegroundColor Green + +Write-Host "[Step 5] Import in K3s..." -ForegroundColor Yellow +docker exec $k3sContainer ctr -n k8s.io images import /tmp/metrics-server.tar +docker exec $k3sContainer rm /tmp/metrics-server.tar +Remove-Item $tarPath +Write-Host "[OK] Imported" -ForegroundColor Green + +Write-Host "[Step 6] Restart metrics-server pod..." -ForegroundColor Yellow +kubectl -n kube-system delete pod -l k8s-app=metrics-server --force --grace-period=0 + +Write-Host "Waiting for pod to start..." -ForegroundColor Cyan +Start-Sleep -Seconds 15 + +for ($i = 1; $i -le 30; $i++) { + $status = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath='{.items[0].status.phase}' 2>$null + $ready = kubectl -n kube-system get pods -l k8s-app=metrics-server -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>$null + + if ($status -eq "Running" -and $ready -eq "True") { + Write-Host "[SUCCESS] metrics-server is ready!" -ForegroundColor Green + break + } + + if ($i % 5 -eq 0) { + Write-Host " Waiting... ($i/30) Current: $status" -ForegroundColor Gray + } + Start-Sleep -Seconds 5 +} + +Write-Host "" +Write-Host "Testing metrics API..." -ForegroundColor Cyan +Start-Sleep -Seconds 5 +kubectl top pods + +Write-Host "" +Write-Host "=== Done ===" -ForegroundColor Green \ No newline at end of file diff --git a/k8s/hpa.yaml b/k8s/hpa.yaml new file mode 100644 index 0000000..0339f19 --- /dev/null +++ b/k8s/hpa.yaml @@ -0,0 +1,69 @@ +# Horizontal Pod Autoscaler - 水平Pod自动扩缩容配置 +# 根据 CPU 和内存使用率自动调整 Pod 副本数量 +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: data-engine-hpa # HPA 资源名称 + namespace: default # 命名空间 +spec: + # 指定要自动扩缩容的目标 Deployment + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: data-engine + + # 副本数量范围 + minReplicas: 1 # 最小副本数:始终保持至少 1 个 Pod 运行 + maxReplicas: 10 # 最大副本数:防止无限扩容,最多扩展到 10 个 Pod + + # 扩缩容触发指标(满足任一条件即触发) + metrics: + # CPU 使用率指标 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 # 当所有 Pod 的平均 CPU 使用率超过 70% 时触发扩容 + + # 内存使用率指标 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 # 当所有 Pod 的平均内存使用率超过 80% 时触发扩容 + + # 扩缩容行为策略 + behavior: + # 向上扩容策略(快速响应负载增加) + scaleUp: + stabilizationWindowSeconds: 60 # 稳定窗口期:60 秒内只基于最高需求决策,防止抖动 + policies: + # 策略1:按固定数量扩容 + - type: Pods + value: 2 # 每次最多增加 2 个 Pod + periodSeconds: 60 # 每 60 秒评估一次 + + # 策略2:按比例扩容 + - type: Percent + value: 50 # 每次最多增加当前副本数的 50% + periodSeconds: 60 # 每 60 秒评估一次 + + selectPolicy: Max # 选择两个策略中扩容幅度更大的方式(更激进) + + # 向下缩容策略(保守缩容,避免频繁波动) + scaleDown: + stabilizationWindowSeconds: 300 # 稳定窗口期:300 秒(5分钟),比扩容更保守 + policies: + # 策略1:按固定数量缩容 + - type: Pods + value: 1 # 每次最多减少 1 个 Pod + periodSeconds: 60 # 每 60 秒评估一次 + + # 策略2:按比例缩容 + - type: Percent + value: 25 # 每次最多减少当前副本数的 25% + periodSeconds: 60 # 每 60 秒评估一次 + + selectPolicy: Min # 选择两个策略中缩容幅度更小的方式(更保守) diff --git a/test-hpa-strong.ps1 b/test-hpa-strong.ps1 new file mode 100644 index 0000000..81f6030 --- /dev/null +++ b/test-hpa-strong.ps1 @@ -0,0 +1,133 @@ +Write-Host "=== HPA Stress Test ===" -ForegroundColor Cyan +Write-Host "" + +# 配置参数 +$url = "http://localhost:30301" +$jobCount = 50 +$requestPerJob = 5000 + +Write-Host "Configuration:" -ForegroundColor Yellow +Write-Host " Target URL: $url" -ForegroundColor White +Write-Host " Concurrent Jobs: $jobCount" -ForegroundColor White +Write-Host " Requests per Job: $requestPerJob" -ForegroundColor White +Write-Host " Total Requests: $($jobCount * $requestPerJob)" -ForegroundColor White +Write-Host "" + +# 清理旧任务 +Write-Host "[1/4] Cleaning up old jobs..." -ForegroundColor Cyan +Get-Job | Where-Object { $_.Name -like "HPA-*" } | Stop-Job | Remove-Job +Start-Sleep -Seconds 2 +Write-Host "[OK] Cleaned" -ForegroundColor Green +Write-Host "" + +# 启动并发任务 +Write-Host "[2/4] Starting $jobCount concurrent jobs..." -ForegroundColor Cyan +$jobs = @() + +for ($i = 1; $i -le $jobCount; $i++) { + try { + $job = Start-Job -Name "HPA-$i" -ScriptBlock { + param($targetUrl, $totalRequests) + $success = 0 + $failed = 0 + + for ($j = 1; $j -le $totalRequests; $j++) { + try { + $response = Invoke-WebRequest -Uri $targetUrl -Method GET -TimeoutSec 3 -UseBasicParsing + if ($response.StatusCode -eq 200) { + $success++ + } + } catch { + $failed++ + } + + # 每 500 个请求短暂休息,避免完全耗尽资源 + if ($j % 500 -eq 0) { + Start-Sleep -Milliseconds 50 + } + } + + return @{ + Success = $success + Failed = $failed + } + } -ArgumentList $url, $requestPerJob + + $jobs += $job + + if ($i % 10 -eq 0) { + Write-Host " Started $i/$jobCount jobs..." -ForegroundColor Gray + } + } catch { + Write-Host " Failed to start job $i : $_" -ForegroundColor Red + } +} + +Write-Host "" +Write-Host "[OK] All $jobCount jobs started" -ForegroundColor Green +Write-Host "" + +# 显示监控命令 +Write-Host "[3/4] Monitoring Commands (run in separate windows):" -ForegroundColor Cyan +Write-Host " Window 1 - HPA Status:" -ForegroundColor Yellow +Write-Host " kubectl get hpa data-engine-hpa -w" -ForegroundColor White +Write-Host "" +Write-Host " Window 2 - Pod Status:" -ForegroundColor Yellow +Write-Host " kubectl get pods -l app=data-engine -w" -ForegroundColor White +Write-Host "" +Write-Host " Window 3 - Resource Usage:" -ForegroundColor Yellow +Write-Host " while(`$true) { Clear-Host; kubectl top pods -l app=data-engine; kubectl get hpa data-engine-hpa; Start-Sleep 3 }" -ForegroundColor White +Write-Host "" + +# 等待用户确认 +Write-Host "[4/4] Load test is running..." -ForegroundColor Green +Write-Host "Press any key to stop all jobs and see results..." -ForegroundColor Yellow +$null = $Host.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown") +Write-Host "" + +# 停止所有任务 +Write-Host "Stopping all jobs..." -ForegroundColor Cyan +Get-Job -Name "HPA-*" | Stop-Job + +# 收集结果 +Write-Host "Collecting results..." -ForegroundColor Cyan +$results = Get-Job -Name "HPA-*" | Receive-Job -Wait +Get-Job -Name "HPA-*" | Remove-Job + +# 统计 +$totalSuccess = 0 +$totalFailed = 0 +foreach ($result in $results) { + $totalSuccess += $result.Success + $totalFailed += $result.Failed +} + +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " TEST RESULTS" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "Total Requests: $($totalSuccess + $totalFailed)" -ForegroundColor White +Write-Host "Successful: $totalSuccess" -ForegroundColor Green +Write-Host "Failed: $totalFailed" -ForegroundColor $(if ($totalFailed -gt 0) { "Red" } else { "Green" }) +Write-Host "" + +# 显示最终状态 +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " FINAL STATUS" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "[HPA Status]" -ForegroundColor Yellow +kubectl get hpa data-engine-hpa +Write-Host "" +Write-Host "[Pod List]" -ForegroundColor Yellow +kubectl get pods -l app=data-engine +Write-Host "" +Write-Host "[Resource Usage]" -ForegroundColor Yellow +kubectl top pods -l app=data-engine 2>$null +if ($LASTEXITCODE -ne 0) { + Write-Host " Metrics not available" -ForegroundColor Gray +} +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Test completed!" -ForegroundColor Green \ No newline at end of file