junhong_cmp_fiber/internal/task/carrier_stop_resume_executor.go

package task

import (
	"context"
	"encoding/json"
	stderrors "errors"
	"strconv"
	"time"

	"go.uber.org/zap"

	"github.com/break/junhong_cmp_fiber/internal/gateway"
	"github.com/break/junhong_cmp_fiber/internal/infrastructure/gatewaylimit"
	"github.com/break/junhong_cmp_fiber/internal/model"
	"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
	iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
	"github.com/break/junhong_cmp_fiber/pkg/constants"
)

// StopResumeCommandService 是停复机指令执行器依赖的停复机服务能力。
type StopResumeCommandService interface {
	DecideCommand(ctx context.Context, cardID uint, scenes []string) (iot_card_svc.CommandDecision, error)
	ExecuteCommand(ctx context.Context, card *model.IotCard, action, reason string) (iot_card_svc.CommandCallResult, error)
	QueryCommandCardStatus(ctx context.Context, card *model.IotCard) (int, bool, error)
	ConfirmCommandState(ctx context.Context, card *model.IotCard, action, reason string) error
	EvaluateDeviceCommands(ctx context.Context, deviceID uint, scenes []string) error
	LegacyDeviceOpLocked(ctx context.Context, deviceID uint) bool
}

// stopResumeTaskTimeout 是一条停复机指令的总执行时长上限（一次判断 + 一次网关调用）。
const stopResumeTaskTimeout = 60 * time.Second

// stopResumeRateLimitedDelay 本账户限流拒绝或等待超时（请求未发出）后的重排延迟，不计失败。
const stopResumeRateLimitedDelay = 5 * time.Second

// StopResumeExecutor 停复机指令执行器：执行前按场景重新判断，只调用一次网关，按错误分类退避与告警，结果未知在队列内确认。
type StopResumeExecutor struct {
	store      *carrierqueue.Store
	settings   *carrierqueue.SettingsProvider
	params     func(carrierID uint) carrierqueue.CarrierParams
	alerts     *carrierqueue.AlertWriter
	service    StopResumeCommandService
	classifier *gatewaylimit.Classifier
	instanceID string
	logger     *zap.Logger
}

// NewStopResumeExecutor 创建停复机指令执行器。
func NewStopResumeExecutor(store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, params func(uint) carrierqueue.CarrierParams,
	alerts *carrierqueue.AlertWriter, service StopResumeCommandService, classifier *gatewaylimit.Classifier, instanceID string, logger *zap.Logger) *StopResumeExecutor {
	if logger == nil {
		logger = zap.NewNop()
	}
	return &StopResumeExecutor{store: store, settings: settings, params: params, alerts: alerts, service: service,
		classifier: classifier, instanceID: instanceID, logger: logger}
}

// srOutcome 是一次执行的收尾决定。
type srOutcome struct {
	stat   string // 计数字段覆盖；statSkip 表示不计数
	done   bool
	nextAt time.Time
	set    map[string]string
	del    []string
}

// statSkip 表示本次执行不计入监控计数（指令不再需要、卡已不存在）。
const statSkip = "-"

func srDone() srOutcome { return srOutcome{done: true} }

func srEnded() srOutcome { return srOutcome{done: true, stat: statSkip} }

func srRetry(at time.Time) srOutcome {
	return srOutcome{nextAt: at, set: map[string]string{}}
}

// Handle 消费一条停复机指令消息；返回后由消费者 FIN。重试与退避全部由 Redis score 表达。
func (e *StopResumeExecutor) Handle(ctx context.Context, body []byte, _ func()) {
	var msg carrierqueue.StopResumeMessage
	if err := json.Unmarshal(body, &msg); err != nil || msg.CardID == 0 || msg.Token == "" {
		e.logger.Warn("停复机指令消息无效，丢弃", zap.ByteString("body", body))
		return
	}
	execValue := msg.Token + "|" + e.instanceID
	got, err := e.store.AcquireStopResumeExec(ctx, msg.CardID, msg.Token, execValue, 2*stopResumeTaskTimeout)
	if err != nil || got != 1 {
		return
	}
	taskCtx, cancel := context.WithTimeout(ctx, stopResumeTaskTimeout)
	outcome := e.execute(taskCtx, msg)
	cancel()

	finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
	defer finishCancel()
	e.countOutcome(finishCtx, msg.CarrierID, outcome)
	result, err := e.store.FinishStopResume(finishCtx, carrierqueue.SRFinish{
		CardID: msg.CardID, CarrierID: msg.CarrierID, Token: msg.Token, Version: msg.Version,
		Done: outcome.done, Unknown: outcome.set[carrierqueue.SRFieldPending] == "1", NextAt: outcome.nextAt, Set: outcome.set, Del: outcome.del,
	})
	switch {
	case err != nil:
		e.logger.Warn("停复机指令收尾失败，等待租约到期后重新领取", zap.Uint("card_id", msg.CardID), zap.Error(err))
	case result == 0:
		e.logger.Warn("停复机指令收尾时处理中标记已不一致", zap.Uint("card_id", msg.CardID), zap.String("metric", "stop_resume.finish_token_mismatch"))
	case result == 2:
		e.logger.Info("停复机指令执行期间已被新指令覆盖，保留新指令", zap.Uint("card_id", msg.CardID), zap.Int64("claimed_version", msg.Version))
	}
}

func (e *StopResumeExecutor) execute(ctx context.Context, msg carrierqueue.StopResumeMessage) (outcome srOutcome) {
	defer func() {
		if r := recover(); r != nil {
			e.logger.Error("停复机指令执行 panic，稍后重试", zap.Uint("card_id", msg.CardID), zap.Any("panic", r))
			outcome = srRetry(time.Now().Add(time.Minute))
		}
	}()
	cmd, err := e.store.ReadCommand(ctx, msg.CardID)
	if err != nil {
		return srRetry(time.Now().Add(time.Minute))
	}
	if !cmd.Exists {
		return srEnded()
	}
	decision, err := e.service.DecideCommand(ctx, msg.CardID, cmd.Scenes)
	if err != nil {
		e.logger.Warn("停复机指令执行前判断失败，稍后重试", zap.Uint("card_id", msg.CardID), zap.Error(err))
		return srRetry(time.Now().Add(time.Minute))
	}
	if decision.Gone {
		return srEnded()
	}
	card := decision.Card
	params := e.params(card.CarrierID)
	settings := e.settings.Current(ctx)
	gatewayCtx := gateway.WithRateLimitWait(ctx, 20*time.Second)

	// 待确认（上次结果未知）：先查上游状态再按规则判断，避免上游已生效而库里未知时直接结束指令。
	clearedPending := false
	if cmd.Pending {
		if outcome, settled := e.confirm(gatewayCtx, card, cmd, settings); settled {
			return outcome
		}
		clearedPending = true
	}
	if decision.Action != cmd.Action {
		// 条件不再满足或已是目标状态：结束指令，不调用上游。
		e.logger.Info("停复机指令执行前重新判断不再需要，结束指令", zap.Uint("card_id", msg.CardID),
			zap.String("action", cmd.Action), zap.String("decided", decision.Action), zap.Strings("scenes", cmd.Scenes))
		e.recovered(cmd, card)
		return srEnded()
	}
	if clearedPending {
		// 确认不是目标状态：未到单卡最小间隔则推后，否则清除待确认后重新调用。
		last, lastErr := e.store.LastStopResumeCall(ctx, card.ID)
		if allowed := last.Add(params.SRMinInterval); lastErr != nil || (!last.IsZero() && allowed.After(time.Now())) {
			if lastErr != nil {
				allowed = time.Now().Add(time.Minute)
			}
			retry := srRetry(allowed)
			retry.del = []string{carrierqueue.SRFieldPending}
			return retry
		}
	}
	result := e.call(gatewayCtx, card, cmd, decision.Reason, params, settings)
	if clearedPending && !result.done && result.set[carrierqueue.SRFieldPending] != "1" {
		result.del = append(result.del, carrierqueue.SRFieldPending)
	}
	return result
}

// confirm 处理待确认指令：查一次卡状态（受限流、不受单卡最小间隔约束）。
// 已是目标状态：按成功回写并结束；若此时规则要求相反动作，写入新指令（版本变大，收尾保留新指令）。
// 查询失败：保持待确认按退避重查，连续未知达阈值告警，不发起停复机。
// 不是目标状态：返回 settled=false，由调用方按最新判断继续。
func (e *StopResumeExecutor) confirm(ctx context.Context, card *model.IotCard, cmd carrierqueue.CommandState,
	settings carrierqueue.Settings) (srOutcome, bool) {
	status, known, err := e.service.QueryCommandCardStatus(ctx, card)
	if gateway.IsRateLimited(err) {
		return srRetry(time.Now().Add(stopResumeRateLimitedDelay)), true
	}
	if err != nil || !known {
		unknown := cmd.Unknown + 1
		retry := srRetry(time.Now().Add(settings.SRBackoffFor(unknown)))
		retry.set[carrierqueue.SRFieldUnknown] = strconv.Itoa(unknown)
		message := "确认查询未得到明确状态"
		if err != nil {
			message = "确认查询失败: " + err.Error()
		}
		retry.set[carrierqueue.SRFieldError] = truncateRunes(constants.StopResumeFailureUnknown+"|"+message, 300)
		if unknown >= settings.SRUnknownAlert {
			e.reportFailure(card, cmd, constants.StopResumeFailureUnknown, message, unknown, cmd.WrittenAt)
		}
		return retry, true
	}
	if status != targetNetworkStatus(cmd.Action) {
		return srOutcome{}, false
	}
	if confirmErr := e.service.ConfirmCommandState(ctx, card, cmd.Action, cmd.Reason); confirmErr != nil {
		e.logger.Warn("确认已是目标状态但回写失败，稍后再确认", zap.Uint("card_id", card.ID), zap.Error(confirmErr))
		return srRetry(time.Now().Add(time.Minute)), true
	}
	e.logger.Info("停复机结果未知，确认查询已是目标状态，按成功结束", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action))
	e.recovered(cmd, card)
	if next, decideErr := e.service.DecideCommand(ctx, card.ID, cmd.Scenes); decideErr == nil && next.Card != nil &&
		next.Action != "" && next.Action != cmd.Action {
		if _, writeErr := e.store.WriteCommand(ctx, carrierqueue.StopResumeCommand{CardID: card.ID, CarrierID: card.CarrierID,
			Action: next.Action, Reason: next.Reason, Scenes: cmd.Scenes, Source: "confirm"}); writeErr != nil {
			e.logger.Warn("确认后按规则写入相反指令失败", zap.Uint("card_id", card.ID), zap.Error(writeErr))
		}
	}
	return srDone(), true
}

// call 调用一次停机或复机并按结果分类处理。
func (e *StopResumeExecutor) call(ctx context.Context, card *model.IotCard, cmd carrierqueue.CommandState, reason string,
	params carrierqueue.CarrierParams, settings carrierqueue.Settings) srOutcome {
	previousCall, previousErr := e.store.LastStopResumeCall(ctx, card.ID)
	now := time.Now()
	if err := e.store.MarkStopResumeCall(ctx, card.ID, now, 2*params.SRMinInterval); err != nil {
		return srRetry(now.Add(time.Minute))
	}
	result, err := e.service.ExecuteCommand(ctx, card, cmd.Action, reason)
	if err == nil {
		e.logger.Info("停复机指令执行成功", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action), zap.String("reason", reason),
			zap.Strings("scenes", cmd.Scenes), zap.String("metric", "stop_resume.success"))
		e.recovered(cmd, card)
		return srDone()
	}
	if gateway.IsRateLimited(err) || stderrors.Is(err, iot_card_svc.ErrCommandBusy) {
		// 本账户限流或该卡复机正在其他路径执行：请求未发出，恢复最近调用时间（读取失败时保守保留），短延迟重排，不计失败。
		if previousErr == nil {
			e.restoreLastCall(ctx, card.ID, previousCall, params)
		}
		return srRetry(time.Now().Add(stopResumeRateLimitedDelay))
	}
	classification := e.classify(ctx, cmd.Action, err)
	// 执行上下文已取消（退出或超时）时请求可能已发出：按结果未知确认，不计失败。
	if result.Unknown || classification.MaybeAccepted || ctx.Err() != nil {
		return e.unknown(card, cmd, err, params, settings)
	}
	message := truncateRunes(classification.Category+"|"+err.Error(), 300)
	switch classification.Category {
	case constants.GatewayClassAlreadyTarget:
		if confirmErr := e.service.ConfirmCommandState(ctx, card, cmd.Action, reason); confirmErr != nil {
			return e.unknown(card, cmd, confirmErr, params, settings)
		}
		e.recovered(cmd, card)
		return srDone()
	case constants.GatewayClassTerminal:
		// 终态不可操作：不重试、结束指令、写告警；停机方向上游已不可用时按目标状态视为已停。
		if cmd.Action == constants.StopResumeActionStop {
			if confirmErr := e.service.ConfirmCommandState(ctx, card, cmd.Action, reason); confirmErr != nil {
				e.logger.Warn("停机遇上游终态，回写已停失败", zap.Uint("card_id", card.ID), zap.Error(confirmErr))
			}
		}
		e.reportFailure(card, cmd, classification.Category, err.Error(), cmd.FailCount+1, now)
		return srOutcome{done: true, stat: constants.StopResumeStatFailed}
	case constants.GatewayClassRateLimitedUpstream:
		retry := srRetry(now.Add(maxDuration(params.SRMinInterval, settings.SRBackoffFor(1))))
		retry.set[carrierqueue.SRFieldError] = message
		return retry
	}
	failCount := cmd.FailCount + 1
	retry := srRetry(now.Add(settings.SRBackoffFor(failCount)))
	retry.set[carrierqueue.SRFieldFail] = strconv.Itoa(failCount)
	retry.set[carrierqueue.SRFieldError] = message
	firstFailed := cmd.FirstFailed
	if firstFailed.IsZero() {
		firstFailed = now
		retry.set[carrierqueue.SRFieldFirst] = strconv.FormatInt(now.UnixMilli(), 10)
	}
	if failCount >= settings.SRAlertFailCount || settings.ImmediateAlert[classification.Category] {
		e.reportFailure(card, cmd, classification.Category, err.Error(), failCount, firstFailed)
	}
	e.logger.Warn("停复机指令执行失败，按退避重试", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action),
		zap.String("category", classification.Category), zap.Int("fail_count", failCount), zap.Time("next_at", retry.nextAt),
		zap.String("metric", "stop_resume.failed"), zap.Error(err))
	return retry
}

// unknown 结果未知：置待确认，确认延迟后先判断再查状态；连续未知达阈值告警，指令不丢弃。
func (e *StopResumeExecutor) unknown(card *model.IotCard, cmd carrierqueue.CommandState, err error, params carrierqueue.CarrierParams,
	settings carrierqueue.Settings) srOutcome {
	unknown := cmd.Unknown + 1
	retry := srRetry(time.Now().Add(params.SRConfirmDelay))
	retry.set[carrierqueue.SRFieldPending] = "1"
	retry.set[carrierqueue.SRFieldUnknown] = strconv.Itoa(unknown)
	retry.set[carrierqueue.SRFieldError] = truncateRunes(constants.StopResumeFailureUnknown+"|"+err.Error(), 300)
	if unknown >= settings.SRUnknownAlert {
		e.reportFailure(card, cmd, constants.StopResumeFailureUnknown, err.Error(), unknown, cmd.WrittenAt)
	}
	e.logger.Warn("停复机结果未知，确认延迟后查状态", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action),
		zap.Int("unknown", unknown), zap.String("metric", "stop_resume.unknown"), zap.Error(err))
	return retry
}

func (e *StopResumeExecutor) classify(ctx context.Context, action string, err error) gatewaylimit.Classification {
	gwErr, ok := gateway.AsGatewayError(err)
	if !ok || gwErr.Category != constants.GatewayErrorUpstream {
		return gatewaylimit.Classification{Category: constants.GatewayClassTransient}
	}
	op := constants.GatewayOpStop
	if action == constants.StopResumeActionStart {
		op = constants.GatewayOpStart
	}
	return e.classifier.Classify(ctx, op, gwErr.ProviderMessage)
}

func (e *StopResumeExecutor) restoreLastCall(ctx context.Context, cardID uint, previous time.Time, params carrierqueue.CarrierParams) {
	if previous.IsZero() {
		e.store.Client().Del(ctx, constants.RedisCardStopResumeLastCallKey(cardID))
		return
	}
	_ = e.store.MarkStopResumeCall(ctx, cardID, previous, 2*params.SRMinInterval)
}

func (e *StopResumeExecutor) reportFailure(card *model.IotCard, cmd carrierqueue.CommandState, category, message string, failCount int, firstFailed time.Time) {
	if firstFailed.IsZero() {
		firstFailed = time.Now()
	}
	e.alerts.ReportFailure(carrierqueue.FailureEvent{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID,
		CheckType: constants.CheckTypeStopResume, Category: category, Message: cmd.Action + ": " + message,
		FailCount: failCount, FirstFailed: firstFailed, At: time.Now()})
}

func (e *StopResumeExecutor) recovered(cmd carrierqueue.CommandState, card *model.IotCard) {
	if card != nil && (cmd.FailCount > 0 || cmd.Unknown > 0) {
		e.alerts.ReportRecovered(card.ID, constants.CheckTypeStopResume)
	}
}

func targetNetworkStatus(action string) int {
	if action == constants.StopResumeActionStop {
		return constants.NetworkStatusOffline
	}
	return constants.NetworkStatusOnline
}

func maxDuration(a, b time.Duration) time.Duration {
	if a > b {
		return a
	}
	return b
}

// countOutcome 累加执行计数：结束为成功，置待确认为结果未知，带失败次数为失败，其余推后为重试。
func (e *StopResumeExecutor) countOutcome(ctx context.Context, carrierID uint, outcome srOutcome) {
	if outcome.stat == statSkip {
		return
	}
	field := constants.StopResumeStatRetry
	switch {
	case outcome.stat != "":
		field = outcome.stat
	case outcome.done:
		field = constants.StopResumeStatSuccess
	case outcome.set[carrierqueue.SRFieldPending] == "1":
		field = constants.StopResumeStatUnknown
	case outcome.set[carrierqueue.SRFieldFail] != "":
		field = constants.StopResumeStatFailed
	}
	e.store.IncrStopResumeStat(ctx, carrierID, field)
}