MiFaZhan
244 lines
7.9 KiB
Go
244 lines
7.9 KiB
Go
package rabbitmq
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/gogf/gf/v2/errors/gerror"
|
|
"github.com/gogf/gf/v2/frame/g"
|
|
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/consts"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/library/rabbitmq"
|
|
"git.5iot.com/iotlcm-public/iotlcm-worker-svr/internal/model"
|
|
)
|
|
|
|
const channelRefreshConsumer = "iotlcm-worker-channel-refresh"
|
|
|
|
type channelRefreshConfig struct {
|
|
Address string `json:"address"`
|
|
Timeout time.Duration `json:"timeout"`
|
|
}
|
|
|
|
type channelRefreshResponse struct {
|
|
Code int `json:"code"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
func (c *Consumers) initChannelRefresh(ctx context.Context) error {
|
|
if !rabbitmq.Enabled(ctx) {
|
|
g.Log().Info(ctx, "channel refresh consumer skipped, RabbitMQ disabled")
|
|
return nil
|
|
}
|
|
if err := rabbitmq.ConsumeJSON[model.ChannelRefreshMessage](
|
|
ctx,
|
|
consts.QueueChannelRefresh,
|
|
channelRefreshConsumer,
|
|
c.handleChannelRefresh,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
g.Log().Infof(ctx, "channel refresh consumer started, queue:%s", consts.QueueChannelRefresh)
|
|
return nil
|
|
}
|
|
|
|
func (c *Consumers) handleChannelRefresh(ctx context.Context, message model.ChannelRefreshMessage) error {
|
|
switch strings.ToLower(strings.TrimSpace(message.Type)) {
|
|
case "channel":
|
|
if strings.TrimSpace(message.ChanKey) == "" {
|
|
g.Log().Errorf(ctx, "channel refresh message dropped, chanKey is empty")
|
|
return nil
|
|
}
|
|
if err := refreshChannelWithRetry(ctx, message.ChanKey); err != nil {
|
|
g.Log().Errorf(ctx, "channel refresh failed, chanKey=%s, err=%+v", message.ChanKey, err)
|
|
return err
|
|
}
|
|
return nil
|
|
case "template":
|
|
if strings.TrimSpace(message.TempKey) == "" {
|
|
g.Log().Errorf(ctx, "channel refresh message dropped, tempKey is empty")
|
|
return nil
|
|
}
|
|
return refreshTemplateChannels(ctx, message.TempKey)
|
|
case "channel_rename":
|
|
oldChanKey := strings.TrimSpace(message.OldChanKey)
|
|
newChanKey := strings.TrimSpace(message.NewChanKey)
|
|
if oldChanKey == "" || newChanKey == "" || oldChanKey == newChanKey {
|
|
g.Log().Errorf(ctx, "channel rename message dropped, oldChanKey=%q newChanKey=%q", oldChanKey, newChanKey)
|
|
return nil
|
|
}
|
|
if err := renameChannelWithRetry(ctx, oldChanKey, newChanKey); err != nil {
|
|
// 返回 error 让消费框架 Nack 并重新入队：rename 各步骤幂等，
|
|
// 重复投递安全；静默 Ack 会导致旧通道实例永久残留，只能重启恢复。
|
|
g.Log().Errorf(ctx, "channel rename failed after retries, will requeue, oldChanKey=%s newChanKey=%s err=%+v", oldChanKey, newChanKey, err)
|
|
return err
|
|
}
|
|
return nil
|
|
default:
|
|
g.Log().Errorf(ctx, "channel refresh message dropped, unsupported type=%s", message.Type)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func renameChannelWithRetry(ctx context.Context, oldChanKey, newChanKey string) error {
|
|
var err error
|
|
for attempt := 1; attempt <= 3; attempt++ {
|
|
err = renameChannel(ctx, oldChanKey, newChanKey)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if attempt == 3 {
|
|
break
|
|
}
|
|
delay := time.Second
|
|
if attempt == 2 {
|
|
delay = 3 * time.Second
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(delay):
|
|
}
|
|
}
|
|
return gerror.Wrapf(err, "channel rename failed after %d attempts, oldChanKey=%s newChanKey=%s", 3, oldChanKey, newChanKey)
|
|
}
|
|
|
|
// renameChannel is ordered so that the old runtime remains available until
|
|
// both upstreams have built the new instance. Card data and ICCID routes are
|
|
// intentionally left untouched for operations staff to migrate manually.
|
|
func renameChannel(ctx context.Context, oldChanKey, newChanKey string) error {
|
|
if err := refreshChannel(ctx, newChanKey); err != nil {
|
|
return gerror.Wrap(err, "refresh new channel failed")
|
|
}
|
|
if err := removeChannelTarget(ctx, "upstream.gateway", oldChanKey); err != nil {
|
|
return gerror.Wrap(err, "remove old gateway channel failed")
|
|
}
|
|
if err := removeChannelTarget(ctx, "upstream.proxy", oldChanKey); err != nil {
|
|
return gerror.Wrap(err, "remove old proxy channel failed")
|
|
}
|
|
g.Log().Infof(ctx, "channel instance rebuilt after rename, oldChanKey=%s newChanKey=%s", oldChanKey, newChanKey)
|
|
return nil
|
|
}
|
|
|
|
func refreshTemplateChannels(ctx context.Context, tempKey string) error {
|
|
var rows []struct {
|
|
ChanKey string `json:"chanKey"`
|
|
}
|
|
if err := g.DB().Model("t_sim_channels").Ctx(ctx).
|
|
Fields("chan_key").
|
|
Where("temp_key", strings.TrimSpace(tempKey)).
|
|
Scan(&rows); err != nil {
|
|
return gerror.Wrap(err, "query template channels failed")
|
|
}
|
|
|
|
var firstErr error
|
|
for _, row := range rows {
|
|
if err := refreshChannelWithRetry(ctx, row.ChanKey); err != nil {
|
|
g.Log().Errorf(ctx, "template channel refresh failed, tempKey=%s, chanKey=%s, err=%+v", tempKey, row.ChanKey, err)
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
}
|
|
}
|
|
return firstErr
|
|
}
|
|
|
|
func refreshChannelWithRetry(ctx context.Context, chanKey string) error {
|
|
var err error
|
|
for attempt := 1; attempt <= 3; attempt++ {
|
|
err = refreshChannel(ctx, chanKey)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if attempt == 3 {
|
|
break
|
|
}
|
|
delay := time.Second
|
|
if attempt == 2 {
|
|
delay = 3 * time.Second
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return gerror.Wrapf(err, "channel refresh canceled after %d attempts, chanKey=%s", attempt, chanKey)
|
|
case <-time.After(delay):
|
|
}
|
|
}
|
|
return gerror.Wrapf(err, "channel refresh failed after %d attempts, chanKey=%s", 3, chanKey)
|
|
}
|
|
|
|
func refreshChannel(ctx context.Context, chanKey string) error {
|
|
chanKey = strings.TrimSpace(chanKey)
|
|
if chanKey == "" {
|
|
return gerror.New("chanKey is required")
|
|
}
|
|
if err := refreshChannelTarget(ctx, "upstream.gateway", chanKey); err != nil {
|
|
return gerror.Wrap(err, "refresh gateway channel failed")
|
|
}
|
|
if err := refreshChannelTarget(ctx, "upstream.proxy", chanKey); err != nil {
|
|
return gerror.Wrap(err, "refresh proxy channel failed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func refreshChannelTarget(ctx context.Context, configKey, chanKey string) error {
|
|
return channelControlTarget(ctx, configKey, "/internal/channel/refresh", chanKey)
|
|
}
|
|
|
|
func removeChannelTarget(ctx context.Context, configKey, chanKey string) error {
|
|
return channelControlTarget(ctx, configKey, "/internal/channel/remove", chanKey)
|
|
}
|
|
|
|
func channelControlTarget(ctx context.Context, configKey, path, chanKey string) error {
|
|
configValue, err := g.Cfg().Get(ctx, configKey)
|
|
if err != nil {
|
|
return gerror.Wrapf(err, "read upstream configuration failed: %s", configKey)
|
|
}
|
|
if configValue == nil || configValue.IsNil() {
|
|
return gerror.Newf("upstream configuration is missing: %s", configKey)
|
|
}
|
|
config := channelRefreshConfig{}
|
|
if err = configValue.Scan(&config); err != nil {
|
|
return gerror.Wrapf(err, "parse upstream configuration failed: %s", configKey)
|
|
}
|
|
address := strings.TrimSpace(config.Address)
|
|
if address == "" {
|
|
return gerror.Newf("upstream address is empty: %s", configKey)
|
|
}
|
|
if config.Timeout <= 0 {
|
|
config.Timeout = 10 * time.Second
|
|
}
|
|
requestCtx, cancel := context.WithTimeout(ctx, config.Timeout)
|
|
defer cancel()
|
|
|
|
client := g.Client()
|
|
client.SetTimeout(config.Timeout)
|
|
controlURL := fmt.Sprintf("http://%s%s?chanKey=%s", address, path, url.QueryEscape(chanKey))
|
|
response, err := client.Post(requestCtx, controlURL, nil)
|
|
if err != nil {
|
|
return gerror.Wrapf(err, "request upstream channel control failed: target=%s path=%s", configKey, path)
|
|
}
|
|
defer response.Close()
|
|
|
|
output := channelRefreshResponse{}
|
|
statusCode := 0
|
|
if response.Response != nil {
|
|
statusCode = response.StatusCode
|
|
}
|
|
if err = json.Unmarshal(response.ReadAll(), &output); err != nil {
|
|
return gerror.Wrapf(err, "parse upstream channel control response failed: target=%s path=%s status=%d", configKey, path, statusCode)
|
|
}
|
|
if output.Code != 0 {
|
|
if output.Message == "" {
|
|
output.Message = "upstream channel control failed"
|
|
}
|
|
return gerror.Newf("upstream channel control failed: target=%s path=%s status=%d message=%s", configKey, path, statusCode, output.Message)
|
|
}
|
|
if statusCode < 200 || statusCode >= 300 {
|
|
return gerror.Newf("upstream channel control failed: target=%s path=%s status=%d", configKey, path, statusCode)
|
|
}
|
|
return nil
|
|
}
|