fix(container): log startup failure before rollback (#13618)

agent/app/service/container_update.go

@@ -1,9 +1,11 @@
 package service
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"sort"
 	"strings"
 	"sync"
@@ -18,6 +20,7 @@ import (
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/stdcopy"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -70,7 +73,7 @@ func (u *ContainerService) ContainerUpdate(req dto.ContainerOperate) error {
 				return createContainerWithDynamicIPFallback(func() (container.CreateResponse, error) {
 					return client.ContainerCreate(ctx, config, hostConf, networkConf, &v1.Platform{}, req.Name)
 				}, networkConf.EndpointsConfig, oldContainer.NetworkSettings)
-			}, newContainerSwitchTaskLogger(t))
+			}, config.Tty, t)
 			if err != nil {
 				return fmt.Errorf("update container failed, err: %v", err)
 			}
@@ -137,7 +140,7 @@ func (u *ContainerService) ContainerUpgrade(req dto.ContainerUpgrade) error {
 				preserveContainerVolumeMounts(hostConf, oldContainer.Mounts)
 				cleanupErr, err := switchContainer(ctx, client, item, oldContainer, func() (container.CreateResponse, error) {
 					return createContainerWithOldNetworks(ctx, client, config, hostConf, oldContainer.NetworkSettings, item)
-				}, newContainerSwitchTaskLogger(t))
+				}, config.Tty, t)
 				if err != nil {
 					upgradeErr := fmt.Errorf("upgrade container %s failed: %w", item, err)
 					upgradeErrors = append(upgradeErrors, upgradeErr)
@@ -166,6 +169,7 @@ type containerSwitchClient interface {
 	ContainerStart(context.Context, string, container.StartOptions) error
 	ContainerRemove(context.Context, string, container.RemoveOptions) error
 	ContainerInspect(context.Context, string) (container.InspectResponse, error)
+	ContainerLogs(context.Context, string, container.LogsOptions) (io.ReadCloser, error)
 	NetworkConnect(context.Context, string, string, *network.EndpointSettings) error
 	NetworkDisconnect(context.Context, string, string, bool) error
 }
@@ -243,17 +247,14 @@ type containerNetworkAttachment struct {
 	isDynamic bool
 }
 
-type containerSwitchLogFunc func(messageKey, containerName string, err error)
-
-func newContainerSwitchTaskLogger(t *task.Task) containerSwitchLogFunc {
-	return func(messageKey, containerName string, err error) {
-		t.LogWithStatus(i18n.GetWithName(messageKey, containerName), err)
-	}
+type containerSwitchLogger interface {
+	LogWithStatus(string, error)
+	Log(string)
 }
 
-func logContainerSwitchStep(logger containerSwitchLogFunc, messageKey, containerName string, err error) {
+func logContainerSwitchStep(logger containerSwitchLogger, messageKey, containerName string, err error) {
 	if logger != nil {
-		logger(messageKey, containerName, err)
+		logger.LogWithStatus(i18n.GetWithName(messageKey, containerName), err)
 	}
 }
 
@@ -264,7 +265,8 @@ func switchContainer(
 	name string,
 	oldContainer container.InspectResponse,
 	createNew func() (container.CreateResponse, error),
-	logger containerSwitchLogFunc,
+	tty bool,
+	logger containerSwitchLogger,
 ) (cleanupErr error, err error) {
 	if oldContainer.ID == "" {
 		return nil, fmt.Errorf("original container ID is empty")
@@ -314,6 +316,7 @@ func switchContainer(
 	}
 	if err := cli.ContainerStart(ctx, created.ID, container.StartOptions{}); err != nil {
 		logContainerSwitchStep(logger, "ContainerStartReplacement", name, err)
+		logContainerStartupLogs(ctx, cli, created.ID, name, tty, logger)
 		rollbackErr := restoreOriginalContainer(ctx, cli, oldContainer.ID, name, wasRunning, created.ID, disconnectedNetworks, logger)
 		return nil, errors.Join(fmt.Errorf("start new container failed: %w", err), rollbackErr)
 	}
@@ -321,6 +324,7 @@ func switchContainer(
 	if wasRunning {
 		if err := waitContainerReady(ctx, cli, created.ID); err != nil {
 			logContainerSwitchStep(logger, "ContainerWaitReplacement", name, err)
+			logContainerStartupLogs(ctx, cli, created.ID, name, tty, logger)
 			rollbackErr := restoreOriginalContainer(ctx, cli, oldContainer.ID, name, wasRunning, created.ID, disconnectedNetworks, logger)
 			return nil, errors.Join(fmt.Errorf("new container readiness check failed: %w", err), rollbackErr)
 		}
@@ -337,8 +341,48 @@ const (
 	containerStartPollInterval  = time.Second
 	containerHealthCheckMinWait = 30 * time.Second
 	containerHealthCheckMaxWait = 10 * time.Minute
+	containerDiagnosticLogTail  = "200"
 )
 
+func logContainerStartupLogs(ctx context.Context, cli containerSwitchClient, containerID, name string, tty bool, logger containerSwitchLogger) {
+	if logger == nil {
+		return
+	}
+	logger.Log(fmt.Sprintf("========== %s ==========", i18n.GetWithName("ContainerStartupDiagnostic", name)))
+	diagnosticCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	reader, err := cli.ContainerLogs(diagnosticCtx, containerID, container.LogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+		Timestamps: true,
+		Tail:       containerDiagnosticLogTail,
+	})
+	if err != nil {
+		logger.Log(i18n.GetWithNameAndErr("ContainerDiagnosticLogsFailed", name, err))
+		return
+	}
+	defer reader.Close()
+
+	var output bytes.Buffer
+	if tty {
+		_, err = io.Copy(&output, reader)
+	} else {
+		_, err = stdcopy.StdCopy(&output, &output, reader)
+	}
+	if err != nil {
+		logger.Log(i18n.GetWithNameAndErr("ContainerDiagnosticLogsFailed", name, err))
+		return
+	}
+	logs := strings.TrimSpace(output.String())
+	logger.Log(fmt.Sprintf("---------- %s ----------", i18n.GetMsgByKey("ContainerRecentLogs")))
+	if logs == "" {
+		logger.Log(i18n.GetMsgByKey("ContainerDiagnosticLogsEmpty"))
+		return
+	}
+	logger.Log(logs)
+}
+
 func waitContainerReady(ctx context.Context, cli containerInspectClient, containerID string) error {
 	info, err := cli.ContainerInspect(ctx, containerID)
 	if err != nil {
@@ -583,7 +627,7 @@ func reconnectOriginalContainerNetworks(ctx context.Context, cli containerSwitch
 	return reconnectErr
 }
 
-func restoreOriginalContainer(ctx context.Context, cli containerSwitchClient, oldContainerID, originalName string, wasRunning bool, newContainer string, disconnectedNetworks []containerNetworkAttachment, logger containerSwitchLogFunc) error {
+func restoreOriginalContainer(ctx context.Context, cli containerSwitchClient, oldContainerID, originalName string, wasRunning bool, newContainer string, disconnectedNetworks []containerNetworkAttachment, logger containerSwitchLogger) error {
 	var rollbackErr error
 	backupName := containerSwitchBackupName(oldContainerID)
 	if newContainer != "" {

agent/i18n/lang/en.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: 'Release original container {{ .name }} network addresse
 ContainerCreateReplacement: 'Create replacement container {{ .name }}'
 ContainerStartReplacement: 'Start replacement container {{ .name }}'
 ContainerWaitReplacement: 'Check replacement container {{ .name }} readiness'
+ContainerStartupDiagnostic: 'Startup failure diagnostics for replacement container {{ .name }}'
+ContainerRecentLogs: 'Recent container logs (last 200 lines)'
+ContainerDiagnosticLogsFailed: 'Failed to load logs for failed container {{ .name }}: {{ .err }}'
+ContainerDiagnosticLogsEmpty: 'No container logs are available'
 ContainerRollbackRestartOld: 'Restart original container {{ .name }}'
 ContainerRollbackRemoveReplacement: 'Remove failed replacement container {{ .name }}'
 ContainerRollbackRenameOld: 'Restore original container {{ .name }} name'

agent/i18n/lang/es-ES.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: 'Liberar las direcciones de red del contenedor original
 ContainerCreateReplacement: 'Crear el contenedor de reemplazo {{ .name }}'
 ContainerStartReplacement: 'Iniciar el contenedor de reemplazo {{ .name }}'
 ContainerWaitReplacement: 'Verificar que el contenedor de reemplazo {{ .name }} esté listo'
+ContainerStartupDiagnostic: 'Diagnóstico del fallo de inicio del contenedor de reemplazo {{ .name }}'
+ContainerRecentLogs: 'Registros recientes del contenedor (últimas 200 líneas)'
+ContainerDiagnosticLogsFailed: 'No se pudieron obtener los registros del contenedor fallido {{ .name }}: {{ .err }}'
+ContainerDiagnosticLogsEmpty: 'No hay registros del contenedor disponibles'
 ContainerRollbackRestartOld: 'Reiniciar el contenedor original {{ .name }}'
 ContainerRollbackRemoveReplacement: 'Eliminar el contenedor de reemplazo fallido {{ .name }}'
 ContainerRollbackRenameOld: 'Restaurar el nombre del contenedor original {{ .name }}'

agent/i18n/lang/fa.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: 'آزادسازی آدرس‌های شبکه کانتی
 ContainerCreateReplacement: 'ایجاد کانتینر جایگزین {{ .name }}'
 ContainerStartReplacement: 'راه‌اندازی کانتینر جایگزین {{ .name }}'
 ContainerWaitReplacement: 'بررسی آماده بودن کانتینر جایگزین {{ .name }}'
+ContainerStartupDiagnostic: 'عیب‌یابی خطای راه‌اندازی کانتینر جایگزین {{ .name }}'
+ContainerRecentLogs: 'گزارش‌های اخیر کانتینر (۲۰۰ خط آخر)'
+ContainerDiagnosticLogsFailed: 'دریافت گزارش‌های کانتینر ناموفق {{ .name }} انجام نشد: {{ .err }}'
+ContainerDiagnosticLogsEmpty: 'هیچ گزارش کانتینری در دسترس نیست'
 ContainerRollbackRestartOld: 'راه‌اندازی دوباره کانتینر اصلی {{ .name }}'
 ContainerRollbackRemoveReplacement: 'حذف کانتینر جایگزین ناموفق {{ .name }}'
 ContainerRollbackRenameOld: 'بازگردانی نام کانتینر اصلی {{ .name }}'

agent/i18n/lang/ja.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: '元のコンテナ {{ .name }} のネットワークア
 ContainerCreateReplacement: '置換コンテナ {{ .name }} を作成'
 ContainerStartReplacement: '置換コンテナ {{ .name }} を起動'
 ContainerWaitReplacement: '置換コンテナ {{ .name }} の準備状態を確認'
+ContainerStartupDiagnostic: '置換コンテナ {{ .name }} の起動失敗診断'
+ContainerRecentLogs: 'コンテナの最新ログ（直近 200 行）'
+ContainerDiagnosticLogsFailed: '失敗したコンテナ {{ .name }} のログ取得に失敗しました: {{ .err }}'
+ContainerDiagnosticLogsEmpty: '利用可能なコンテナログがありません'
 ContainerRollbackRestartOld: '元のコンテナ {{ .name }} を再起動'
 ContainerRollbackRemoveReplacement: '起動に失敗した置換コンテナ {{ .name }} を削除'
 ContainerRollbackRenameOld: '元のコンテナ {{ .name }} の名前を復元'

agent/i18n/lang/ko.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: '기존 컨테이너 {{ .name }} 네트워크 주소 해
 ContainerCreateReplacement: '대체 컨테이너 {{ .name }} 생성'
 ContainerStartReplacement: '대체 컨테이너 {{ .name }} 시작'
 ContainerWaitReplacement: '대체 컨테이너 {{ .name }} 준비 상태 확인'
+ContainerStartupDiagnostic: '대체 컨테이너 {{ .name }} 시작 실패 진단'
+ContainerRecentLogs: '최근 컨테이너 로그(마지막 200줄)'
+ContainerDiagnosticLogsFailed: '실패한 컨테이너 {{ .name }} 로그를 가져오지 못했습니다: {{ .err }}'
+ContainerDiagnosticLogsEmpty: '사용 가능한 컨테이너 로그가 없습니다'
 ContainerRollbackRestartOld: '기존 컨테이너 {{ .name }} 다시 시작'
 ContainerRollbackRemoveReplacement: '시작에 실패한 대체 컨테이너 {{ .name }} 제거'
 ContainerRollbackRenameOld: '기존 컨테이너 {{ .name }} 이름 복원'

agent/i18n/lang/lo.yaml

@@ -582,6 +582,10 @@ ContainerDisconnectOld: 'ປ່ອຍທີ່ຢູ່ເຄືອຂ່າຍ
 ContainerCreateReplacement: 'ສ້າງຄອນເທນເນີທົດແທນ {{ .name }}'
 ContainerStartReplacement: 'ເລີ່ມຄອນເທນເນີທົດແທນ {{ .name }}'
 ContainerWaitReplacement: 'ກວດສອບຄອນເທນເນີທົດແທນ {{ .name }} ວ່າພ້ອມແລ້ວ'
+ContainerStartupDiagnostic: 'ການວິນິດໄສການເລີ່ມລົ້ມເຫຼວຂອງຄອນເທນເນີທົດແທນ {{ .name }}'
+ContainerRecentLogs: 'ບັນທຶກຄອນເທນເນີຫຼ້າສຸດ (200 ແຖວ)'
+ContainerDiagnosticLogsFailed: 'ບໍ່ສາມາດຮັບບັນທຶກຂອງຄອນເທນເນີທີ່ລົ້ມເຫຼວ {{ .name }}: {{ .err }}'
+ContainerDiagnosticLogsEmpty: 'ບໍ່ມີບັນທຶກຄອນເທນເນີ'
 ContainerRollbackRestartOld: 'ເລີ່ມຄອນເທນເນີເດີມ {{ .name }} ຄືນໃໝ່'
 ContainerRollbackRemoveReplacement: 'ລຶບຄອນເທນເນີທົດແທນ {{ .name }} ທີ່ລົ້ມເຫຼວ'
 ContainerRollbackRenameOld: 'ກູ້ຄືນຊື່ຄອນເທນເນີເດີມ {{ .name }}'

agent/i18n/lang/ms.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: 'Lepaskan alamat rangkaian bekas asal {{ .name }}'
 ContainerCreateReplacement: 'Cipta bekas ganti {{ .name }}'
 ContainerStartReplacement: 'Mulakan bekas ganti {{ .name }}'
 ContainerWaitReplacement: 'Periksa kesediaan bekas ganti {{ .name }}'
+ContainerStartupDiagnostic: 'Diagnostik kegagalan permulaan bekas ganti {{ .name }}'
+ContainerRecentLogs: 'Log bekas terkini (200 baris terakhir)'
+ContainerDiagnosticLogsFailed: 'Gagal mendapatkan log bekas yang gagal {{ .name }}: {{ .err }}'
+ContainerDiagnosticLogsEmpty: 'Tiada log bekas tersedia'
 ContainerRollbackRestartOld: 'Mulakan semula bekas asal {{ .name }}'
 ContainerRollbackRemoveReplacement: 'Alih keluar bekas ganti yang gagal {{ .name }}'
 ContainerRollbackRenameOld: 'Pulihkan nama bekas asal {{ .name }}'

agent/i18n/lang/pt-BR.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: 'Liberar os endereços de rede do contêiner original {{
 ContainerCreateReplacement: 'Criar o contêiner substituto {{ .name }}'
 ContainerStartReplacement: 'Iniciar o contêiner substituto {{ .name }}'
 ContainerWaitReplacement: 'Verificar se o contêiner substituto {{ .name }} está pronto'
+ContainerStartupDiagnostic: 'Diagnóstico da falha de inicialização do contêiner substituto {{ .name }}'
+ContainerRecentLogs: 'Logs recentes do contêiner (últimas 200 linhas)'
+ContainerDiagnosticLogsFailed: 'Falha ao obter os logs do contêiner com falha {{ .name }}: {{ .err }}'
+ContainerDiagnosticLogsEmpty: 'Não há logs de contêiner disponíveis'
 ContainerRollbackRestartOld: 'Reiniciar o contêiner original {{ .name }}'
 ContainerRollbackRemoveReplacement: 'Remover o contêiner substituto com falha {{ .name }}'
 ContainerRollbackRenameOld: 'Restaurar o nome do contêiner original {{ .name }}'

agent/i18n/lang/ru.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: 'Освобождение сетевых адресов
 ContainerCreateReplacement: 'Создание нового контейнера {{ .name }}'
 ContainerStartReplacement: 'Запуск нового контейнера {{ .name }}'
 ContainerWaitReplacement: 'Проверка готовности нового контейнера {{ .name }}'
+ContainerStartupDiagnostic: 'Диагностика сбоя запуска нового контейнера {{ .name }}'
+ContainerRecentLogs: 'Последние журналы контейнера (200 строк)'
+ContainerDiagnosticLogsFailed: 'Не удалось получить журналы сбойного контейнера {{ .name }}: {{ .err }}'
+ContainerDiagnosticLogsEmpty: 'Журналы контейнера отсутствуют'
 ContainerRollbackRestartOld: 'Повторный запуск исходного контейнера {{ .name }}'
 ContainerRollbackRemoveReplacement: 'Удаление неудачно созданного контейнера {{ .name }}'
 ContainerRollbackRenameOld: 'Восстановление имени исходного контейнера {{ .name }}'

agent/i18n/lang/tr.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: 'Eski {{ .name }} konteynerinin ağ adreslerini serbest
 ContainerCreateReplacement: 'Yeni {{ .name }} konteynerini oluştur'
 ContainerStartReplacement: 'Yeni {{ .name }} konteynerini başlat'
 ContainerWaitReplacement: 'Yeni {{ .name }} konteynerinin hazır olduğunu kontrol et'
+ContainerStartupDiagnostic: 'Yeni {{ .name }} konteynerinin başlatma hatası tanılaması'
+ContainerRecentLogs: 'Son konteyner günlükleri (son 200 satır)'
+ContainerDiagnosticLogsFailed: 'Başarısız {{ .name }} konteynerinin günlükleri alınamadı: {{ .err }}'
+ContainerDiagnosticLogsEmpty: 'Kullanılabilir konteyner günlüğü yok'
 ContainerRollbackRestartOld: 'Eski {{ .name }} konteynerini yeniden başlat'
 ContainerRollbackRemoveReplacement: 'Başlatılamayan yeni {{ .name }} konteynerini kaldır'
 ContainerRollbackRenameOld: 'Eski {{ .name }} konteynerinin adını geri yükle'

agent/i18n/lang/zh-Hant.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: '釋放原容器 {{ .name }} 的網路位址'
 ContainerCreateReplacement: '建立替換容器 {{ .name }}'
 ContainerStartReplacement: '啟動替換容器 {{ .name }}'
 ContainerWaitReplacement: '檢查替換容器 {{ .name }} 是否就緒'
+ContainerStartupDiagnostic: '替換容器 {{ .name }} 啟動失敗診斷'
+ContainerRecentLogs: '最近 200 行容器日誌'
+ContainerDiagnosticLogsFailed: '取得失敗容器 {{ .name }} 日誌失敗：{{ .err }}'
+ContainerDiagnosticLogsEmpty: '容器沒有可用日誌'
 ContainerRollbackRestartOld: '重新啟動原容器 {{ .name }}'
 ContainerRollbackRemoveReplacement: '刪除啟動失敗的替換容器 {{ .name }}'
 ContainerRollbackRenameOld: '恢復原容器 {{ .name }} 的名稱'

agent/i18n/lang/zh.yaml

@@ -591,6 +591,10 @@ ContainerDisconnectOld: "释放原容器 {{ .name }} 的网络地址"
 ContainerCreateReplacement: "创建替换容器 {{ .name }}"
 ContainerStartReplacement: "启动替换容器 {{ .name }}"
 ContainerWaitReplacement: "检查替换容器 {{ .name }} 是否就绪"
+ContainerStartupDiagnostic: "替换容器 {{ .name }} 启动失败诊断"
+ContainerRecentLogs: "最近 200 行容器日志"
+ContainerDiagnosticLogsFailed: "获取失败容器 {{ .name }} 日志失败：{{ .err }}"
+ContainerDiagnosticLogsEmpty: "容器没有可用日志"
 ContainerRollbackRestartOld: "重新启动原容器 {{ .name }}"
 ContainerRollbackRemoveReplacement: "删除启动失败的替换容器 {{ .name }}"
 ContainerRollbackRenameOld: "恢复原容器 {{ .name }} 的名称"