《游戏服务端架构·平台化与架构治理》11.2 智能客服与内容审核

// file: internal/aimoderation/pipeline.go
// tier: L1
// verify: go test ./internal/aimoderation/
package aimoderation

import (
	"errors"
	"strings"
)

// Decision is the outcome of routing one moderation score.
type Decision string

const (
	// Pass means publish without a human.
	Pass Decision = "pass"
	// Review means a human must look before the content is published or
	// removed. Review is the safe default for anything uncertain.
	Review Decision = "review"
	// Reject means block automatically. It is reserved for high-confidence,
	// non-protected cases.
	Reject Decision = "reject"
)

var (
	ErrScoreRange     = errors.New("aimoderation: score out of range")
	ErrPolicyInvalid  = errors.New("aimoderation: invalid policy")
	ErrMissingVersion = errors.New("aimoderation: missing policy version")
)

// Policy is a versioned routing table. Every result carries the version so a
// bad release can be traced and rolled back without guessing.
type Policy struct {
	Version     string
	PassBelow   float64
	RejectAbove float64
	Protected   []string
	RepeatLimit int
}

// Validate rejects ambiguous bands. An overlapping or inverted band is a
// configuration incident, not something to paper over at runtime.
func (p Policy) Validate() error {
	if strings.TrimSpace(p.Version) == "" {
		return ErrMissingVersion
	}
	if p.PassBelow < 0 || p.RejectAbove > 1 || p.PassBelow >= p.RejectAbove {
		return ErrPolicyInvalid
	}
	if p.RepeatLimit < 0 {
		return ErrPolicyInvalid
	}
	return nil
}

// Item is one piece of user-generated content. Score is the model's
// violation probability in [0,1]; higher means more likely to violate.
type Item struct {
	ID       string
	Category string
	Score    float64
	Minor    bool
	Repeat   int
}

// Verdict explains why a decision was made. The reason is part of the audit
// record and must not be a free-form model output.
type Verdict struct {
	Decision Decision
	Reason   string
	Policy   string
}

// Route maps a score to pass, review, or reject. The ordering is deliberate:
// repeat offenders, minors, and protected categories are checked before the
// automatic reject band, because those cases carry irreversible risk.
func (p Policy) Route(it Item) (Verdict, error) {
	if err := p.Validate(); err != nil {
		return Verdict{}, err
	}
	if it.Score < 0 || it.Score > 1 {
		return Verdict{}, ErrScoreRange
	}

	v := Verdict{Policy: p.Version}
	if it.Score <= p.PassBelow {
		if p.RepeatLimit > 0 && it.Repeat >= p.RepeatLimit {
			v.Decision = Review
			v.Reason = "repeat_offender"
			return v, nil
		}
		v.Decision = Pass
		v.Reason = "below_pass_threshold"
		return v, nil
	}

	if it.Minor {
		v.Decision = Review
		v.Reason = "minor_protection"
		return v, nil
	}
	if p.protected(it.Category) {
		v.Decision = Review
		v.Reason = "protected_category"
		return v, nil
	}
	if it.Score >= p.RejectAbove {
		v.Decision = Reject
		v.Reason = "above_reject_threshold"
		return v, nil
	}

	v.Decision = Review
	v.Reason = "low_confidence"
	return v, nil
}

// protected matches category names case-insensitively. A miss here means an
// automatic reject on content that policy says must be seen by a human.
func (p Policy) protected(category string) bool {
	for _, c := range p.Protected {
		if strings.EqualFold(strings.TrimSpace(c), strings.TrimSpace(category)) {
			return true
		}
	}
	return false
}

// file: internal/aimoderation/appeal.go
// tier: L1
// verify: go test ./internal/aimoderation/
package aimoderation

import (
	"errors"
	"fmt"
	"strings"
	"time"
)

// AppealState is the lifecycle state of one user appeal. Only Open can move;
// every other state is terminal so a replay cannot silently reopen a case.
type AppealState string

const (
	AppealOpen     AppealState = "open"
	AppealUpheld   AppealState = "upheld"
	AppealReversed AppealState = "reversed"
	AppealExpired  AppealState = "expired"
)

var (
	ErrAppealClosed  = errors.New("aimoderation: appeal already closed")
	ErrAppealExpired = errors.New("aimoderation: appeal window expired")
	ErrNoReviewer    = errors.New("aimoderation: reviewer required")
	ErrBadWindow     = errors.New("aimoderation: window must be positive")
)

// Appeal records who decided and when. ResolvedAt is evidence for the
// compliance report; it is never inferred from the request time.
type Appeal struct {
	ItemID     string
	State      AppealState
	Deadline   time.Time
	Reviewer   string
	ResolvedAt time.Time
}

// NewAppeal opens a window. A positive, bounded window is mandatory: an
// appeal that never expires creates an unbounded review queue.
func NewAppeal(itemID string, createdAt time.Time, window time.Duration) (*Appeal, error) {
	if strings.TrimSpace(itemID) == "" {
		return nil, fmt.Errorf("aimoderation: appeal needs an item id")
	}
	if window <= 0 {
		return nil, ErrBadWindow
	}
	return &Appeal{
		ItemID:   itemID,
		State:    AppealOpen,
		Deadline: createdAt.Add(window),
	}, nil
}

// Resolve closes an open appeal. The reviewer must be named; an empty value
// would make the audit record useless. reverse=true means the original
// decision is overturned, not that the content is deleted.
func (a *Appeal) Resolve(now time.Time, reviewer string, reverse bool) error {
	if a.State != AppealOpen {
		return ErrAppealClosed
	}
	if now.After(a.Deadline) {
		return ErrAppealExpired
	}
	if strings.TrimSpace(reviewer) == "" {
		return ErrNoReviewer
	}

	a.Reviewer = reviewer
	a.ResolvedAt = now
	if reverse {
		a.State = AppealReversed
	} else {
		a.State = AppealUpheld
	}
	return nil
}

// Expire closes an appeal that nobody handled in time. It is a separate
// operation from Resolve so a scheduled job cannot claim a human reviewed it.
func (a *Appeal) Expire(now time.Time) error {
	if a.State != AppealOpen {
		return ErrAppealClosed
	}
	if !now.After(a.Deadline) {
		return fmt.Errorf("aimoderation: appeal window still open")
	}
	a.State = AppealExpired
	a.ResolvedAt = now
	return nil
}