增加熔断策略
This commit is contained in:
@@ -252,11 +252,11 @@ func updateResponseTimeStats(cbInfo *CircuitBreakerInfo, duration time.Duration,
|
|||||||
|
|
||||||
// formatUnixTime 格式化Unix时间戳
|
// formatUnixTime 格式化Unix时间戳
|
||||||
func formatUnixTime(timestamp int64) string {
|
func formatUnixTime(timestamp int64) string {
|
||||||
if timestamp > 0 {
|
if timestamp <= 0 {
|
||||||
return time.Unix(timestamp, 0).Format("2006-01-02 15:04:05")
|
|
||||||
}
|
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
return time.Unix(timestamp, 0).Format("2006-01-02 15:04:05")
|
||||||
|
}
|
||||||
|
|
||||||
// InitCircuitBreaker 初始化Sentinel熔断器
|
// InitCircuitBreaker 初始化Sentinel熔断器
|
||||||
func InitCircuitBreaker() error {
|
func InitCircuitBreaker() error {
|
||||||
@@ -445,20 +445,6 @@ func getAllowedIPsAndCIDRs() (map[string]bool, []*net.IPNet) {
|
|||||||
return allowedAdminIPsMap, allowedAdminCIDRs
|
return allowedAdminIPsMap, allowedAdminCIDRs
|
||||||
}
|
}
|
||||||
|
|
||||||
// getAllowedIPs 获取允许的IP列表(带锁保护,兼容旧代码)
|
|
||||||
func getAllowedIPs() map[string]bool {
|
|
||||||
allowedAdminIPsMutex.RLock()
|
|
||||||
defer allowedAdminIPsMutex.RUnlock()
|
|
||||||
return allowedAdminIPsMap
|
|
||||||
}
|
|
||||||
|
|
||||||
// getAllowedCIDRs 获取允许的CIDR列表(带锁保护,兼容旧代码)
|
|
||||||
func getAllowedCIDRs() []*net.IPNet {
|
|
||||||
allowedAdminCIDRsMutex.RLock()
|
|
||||||
defer allowedAdminCIDRsMutex.RUnlock()
|
|
||||||
return allowedAdminCIDRs
|
|
||||||
}
|
|
||||||
|
|
||||||
// reset 重置所有指标到初始状态
|
// reset 重置所有指标到初始状态
|
||||||
func (m *CircuitBreakerMetrics) reset() {
|
func (m *CircuitBreakerMetrics) reset() {
|
||||||
m.TotalRequests.Store(0)
|
m.TotalRequests.Store(0)
|
||||||
@@ -533,9 +519,9 @@ func (cb *CircuitBreakerInfo) updateWindowStats(isSuccess bool, ctx context.Cont
|
|||||||
// 计算当前窗口内的成功率
|
// 计算当前窗口内的成功率
|
||||||
total := cb.Metrics.WindowRequests.Load()
|
total := cb.Metrics.WindowRequests.Load()
|
||||||
failures := cb.Metrics.WindowFailures.Load()
|
failures := cb.Metrics.WindowFailures.Load()
|
||||||
if total > 0 {
|
if total >= 10 { // 有足够样本时才记录
|
||||||
successRate := float64(total-failures) / float64(total)
|
successRate := float64(total-failures) / float64(total)
|
||||||
if successRate < 0.5 && total >= 10 { // 如果成功率低于50%且有足够样本
|
if successRate < 0.5 { // 如果成功率低于50%
|
||||||
g.Log().Warningf(ctx, "熔断器 %s 窗口内成功率较低: %.2f%%, total=%d, failures=%d",
|
g.Log().Warningf(ctx, "熔断器 %s 窗口内成功率较低: %.2f%%, total=%d, failures=%d",
|
||||||
cb.ResourceName, successRate*100, total, failures)
|
cb.ResourceName, successRate*100, total, failures)
|
||||||
}
|
}
|
||||||
@@ -614,28 +600,25 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig)
|
|||||||
}
|
}
|
||||||
|
|
||||||
var rule []*circuitbreaker.Rule
|
var rule []*circuitbreaker.Rule
|
||||||
if config.EnableSlidingWindow {
|
baseRule := &circuitbreaker.Rule{
|
||||||
rule = []*circuitbreaker.Rule{{
|
|
||||||
Resource: resourceName,
|
Resource: resourceName,
|
||||||
Strategy: circuitbreaker.SlowRequestRatio,
|
|
||||||
RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()),
|
RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()),
|
||||||
MinRequestAmount: uint64(config.MinRequestAmount),
|
MinRequestAmount: uint64(config.MinRequestAmount),
|
||||||
StatIntervalMs: uint32(config.StatIntervalMs),
|
StatIntervalMs: uint32(config.StatIntervalMs),
|
||||||
StatSlidingWindowBucketCount: 10,
|
|
||||||
MaxAllowedRtMs: uint64(config.SlowRequestThresholdParsed.Milliseconds()),
|
|
||||||
Threshold: threshold,
|
|
||||||
}}
|
|
||||||
} else {
|
|
||||||
rule = []*circuitbreaker.Rule{{
|
|
||||||
Resource: resourceName,
|
|
||||||
Strategy: circuitbreaker.ErrorCount,
|
|
||||||
RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()),
|
|
||||||
MinRequestAmount: uint64(config.MinRequestAmount),
|
|
||||||
StatIntervalMs: uint32(config.StatIntervalMs),
|
|
||||||
Threshold: float64(config.MaxFailures),
|
|
||||||
}}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config.EnableSlidingWindow {
|
||||||
|
baseRule.Strategy = circuitbreaker.SlowRequestRatio
|
||||||
|
baseRule.StatSlidingWindowBucketCount = 10
|
||||||
|
baseRule.MaxAllowedRtMs = uint64(config.SlowRequestThresholdParsed.Milliseconds())
|
||||||
|
baseRule.Threshold = threshold
|
||||||
|
} else {
|
||||||
|
baseRule.Strategy = circuitbreaker.ErrorCount
|
||||||
|
baseRule.Threshold = float64(config.MaxFailures)
|
||||||
|
}
|
||||||
|
|
||||||
|
rule = []*circuitbreaker.Rule{baseRule}
|
||||||
|
|
||||||
if _, err := circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}); err != nil {
|
if _, err := circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}); err != nil {
|
||||||
return fmt.Errorf("清空熔断规则失败: %v", err)
|
return fmt.Errorf("清空熔断规则失败: %v", err)
|
||||||
}
|
}
|
||||||
@@ -849,14 +832,14 @@ func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitB
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
msg := fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName)
|
||||||
switch reason {
|
switch reason {
|
||||||
case "blocked":
|
case "blocked":
|
||||||
r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 熔断保护中,请稍后再试", serviceName))
|
msg = fmt.Sprintf("服务 '%s' 熔断保护中,请稍后再试", serviceName)
|
||||||
case "distributed":
|
case "distributed":
|
||||||
r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 分布式熔断中", serviceName))
|
msg = fmt.Sprintf("服务 '%s' 分布式熔断中", serviceName)
|
||||||
default:
|
|
||||||
r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName))
|
|
||||||
}
|
}
|
||||||
|
r.Response.WriteStatusExit(503, msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// isSuccessStatusCode 判断HTTP状态码是否成功
|
// isSuccessStatusCode 判断HTTP状态码是否成功
|
||||||
@@ -877,16 +860,16 @@ func extractServiceName(path string) string {
|
|||||||
if path == "" {
|
if path == "" {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
parts := strings.Split(path, "/")
|
|
||||||
if len(parts) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
serviceName := parts[0]
|
|
||||||
|
|
||||||
if strings.Contains(serviceName, "%") {
|
// 获取第一个路径段
|
||||||
if decoded, err := pathUnescape(serviceName); err == nil {
|
if idx := strings.Index(path, "/"); idx > 0 {
|
||||||
serviceName = decoded
|
path = path[:idx]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 解码URL编码(简化版)
|
||||||
|
serviceName := path
|
||||||
|
if strings.Contains(serviceName, "%") {
|
||||||
|
serviceName = urlDecode(serviceName)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := circuitBreakerConfigs.Load(serviceName); ok {
|
if _, ok := circuitBreakerConfigs.Load(serviceName); ok {
|
||||||
@@ -895,31 +878,24 @@ func extractServiceName(path string) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// pathUnescape 路径片段的URL解码
|
// urlDecode 简单的URL解码
|
||||||
func pathUnescape(s string) (string, error) {
|
func urlDecode(s string) string {
|
||||||
var builder strings.Builder
|
result := make([]byte, 0, len(s))
|
||||||
builder.Grow(len(s))
|
|
||||||
|
|
||||||
for i := 0; i < len(s); i++ {
|
for i := 0; i < len(s); i++ {
|
||||||
switch s[i] {
|
if s[i] == '%' && i+2 < len(s) {
|
||||||
case '%':
|
if high := hexDigit(s[i+1]); high != 0xFF {
|
||||||
if i+2 >= len(s) {
|
if low := hexDigit(s[i+2]); low != 0xFF {
|
||||||
builder.WriteByte(s[i])
|
result = append(result, (high<<4)|low)
|
||||||
|
i += 2
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
high := hexDigit(s[i+1])
|
|
||||||
low := hexDigit(s[i+2])
|
|
||||||
if high == 0xFF || low == 0xFF {
|
|
||||||
builder.WriteByte(s[i])
|
|
||||||
} else {
|
|
||||||
builder.WriteByte((high << 4) | low)
|
|
||||||
i += 2
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
builder.WriteByte(s[i])
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return builder.String(), nil
|
result = append(result, s[i])
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
func hexDigit(c byte) byte {
|
func hexDigit(c byte) byte {
|
||||||
@@ -978,11 +954,11 @@ func filterServiceNames(services map[string]interface{}) []string {
|
|||||||
// isCircuitBreakerOpenInDistributed 检查分布式熔断状态
|
// isCircuitBreakerOpenInDistributed 检查分布式熔断状态
|
||||||
func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) bool {
|
func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) bool {
|
||||||
key := "circuit_breaker:" + resourceName + ":state"
|
key := "circuit_breaker:" + resourceName + ":state"
|
||||||
redis := g.Redis()
|
redisClient := g.Redis()
|
||||||
if redis == nil {
|
if redisClient == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
value, err := redis.Get(ctx, key)
|
value, err := redisClient.Get(ctx, key)
|
||||||
if err != nil || value.IsNil() {
|
if err != nil || value.IsNil() {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -1219,9 +1195,9 @@ func resetSingleService(r *ghttp.Request, serviceName string) error {
|
|||||||
if configVal, ok := circuitBreakerConfigs.Load(serviceName); ok {
|
if configVal, ok := circuitBreakerConfigs.Load(serviceName); ok {
|
||||||
config, ok := configVal.(*CircuitBreakerConfig)
|
config, ok := configVal.(*CircuitBreakerConfig)
|
||||||
if ok && config.DistributedTTL > 0 {
|
if ok && config.DistributedTTL > 0 {
|
||||||
redis := g.Redis()
|
redisClient := g.Redis()
|
||||||
if redis != nil {
|
if redisClient != nil {
|
||||||
if _, err := redis.Del(r.GetCtx(), "circuit_breaker:"+resourceName+":state"); err != nil {
|
if _, err := redisClient.Del(r.GetCtx(), "circuit_breaker:"+resourceName+":state"); err != nil {
|
||||||
g.Log().Warningf(r.GetCtx(), "清除分布式熔断状态失败: %s, error: %v", resourceName, err)
|
g.Log().Warningf(r.GetCtx(), "清除分布式熔断状态失败: %s, error: %v", resourceName, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user