Docs i18n: harden doc-mode pipeline

2026-02-19 18:39:20 -05:00 · 2026-02-01 22:47:33 +01:00
parent da9f28d270
commit e70984745b
7 changed files with 760 additions and 37 deletions
--- a/scripts/docs-i18n/doc_mode.go
+++ b/scripts/docs-i18n/doc_mode.go
@@ -0,0 +1,272 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	frontmatterTagStart = "<frontmatter>"
+	frontmatterTagEnd   = "</frontmatter>"
+	bodyTagStart        = "<body>"
+	bodyTagEnd          = "</body>"
+)
+
+func processFileDoc(ctx context.Context, translator *PiTranslator, docsRoot, filePath, srcLang, tgtLang string, overwrite bool) (bool, error) {
+	absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
+	if err != nil {
+		return false, err
+	}
+
+	content, err := os.ReadFile(absPath)
+	if err != nil {
+		return false, err
+	}
+	currentHash := hashBytes(content)
+
+	outputPath := filepath.Join(docsRoot, tgtLang, relPath)
+	if !overwrite {
+		skip, err := shouldSkipDoc(outputPath, currentHash)
+		if err != nil {
+			return false, err
+		}
+		if skip {
+			return true, nil
+		}
+	}
+
+	sourceFront, sourceBody := splitFrontMatter(string(content))
+	frontData := map[string]any{}
+	if strings.TrimSpace(sourceFront) != "" {
+		if err := yaml.Unmarshal([]byte(sourceFront), &frontData); err != nil {
+			return false, fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
+		}
+	}
+	frontTemplate, markers := buildFrontmatterTemplate(frontData)
+	taggedInput := formatTaggedDocument(frontTemplate, sourceBody)
+
+	translatedDoc, err := translator.TranslateRaw(ctx, taggedInput, srcLang, tgtLang)
+	if err != nil {
+		return false, fmt.Errorf("translate failed (%s): %w", relPath, err)
+	}
+
+	translatedFront, translatedBody, err := parseTaggedDocument(translatedDoc)
+	if err != nil {
+		return false, fmt.Errorf("tagged output invalid for %s: %w", relPath, err)
+	}
+	if sourceFront != "" && strings.TrimSpace(translatedFront) == "" {
+		return false, fmt.Errorf("translation removed frontmatter for %s", relPath)
+	}
+	if err := applyFrontmatterTranslations(frontData, markers, translatedFront); err != nil {
+		return false, fmt.Errorf("frontmatter translation failed for %s: %w", relPath, err)
+	}
+
+	updatedFront, err := encodeFrontMatter(frontData, relPath, content)
+	if err != nil {
+		return false, err
+	}
+
+	if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
+		return false, err
+	}
+
+	output := updatedFront + translatedBody
+	return false, os.WriteFile(outputPath, []byte(output), 0o644)
+}
+
+func formatTaggedDocument(frontMatter, body string) string {
+	return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", frontmatterTagStart, frontMatter, frontmatterTagEnd, bodyTagStart, body, bodyTagEnd)
+}
+
+func parseTaggedDocument(text string) (string, string, error) {
+	frontStart := strings.Index(text, frontmatterTagStart)
+	if frontStart == -1 {
+		return "", "", fmt.Errorf("missing %s", frontmatterTagStart)
+	}
+	frontStart += len(frontmatterTagStart)
+	frontEnd := strings.Index(text[frontStart:], frontmatterTagEnd)
+	if frontEnd == -1 {
+		return "", "", fmt.Errorf("missing %s", frontmatterTagEnd)
+	}
+	frontEnd += frontStart
+
+	bodyStart := strings.Index(text[frontEnd:], bodyTagStart)
+	if bodyStart == -1 {
+		return "", "", fmt.Errorf("missing %s", bodyTagStart)
+	}
+	bodyStart += frontEnd + len(bodyTagStart)
+	bodyEnd := strings.Index(text[bodyStart:], bodyTagEnd)
+	if bodyEnd == -1 {
+		return "", "", fmt.Errorf("missing %s", bodyTagEnd)
+	}
+	bodyEnd += bodyStart
+
+	prefix := strings.TrimSpace(text[:frontStart-len(frontmatterTagStart)])
+	suffix := strings.TrimSpace(text[bodyEnd+len(bodyTagEnd):])
+	if prefix != "" || suffix != "" {
+		return "", "", fmt.Errorf("unexpected text outside tagged sections")
+	}
+
+	frontMatter := trimTagNewlines(text[frontStart:frontEnd])
+	body := trimTagNewlines(text[bodyStart:bodyEnd])
+	return frontMatter, body, nil
+}
+
+func trimTagNewlines(value string) string {
+	value = strings.TrimPrefix(value, "\n")
+	value = strings.TrimSuffix(value, "\n")
+	return value
+}
+
+type frontmatterMarker struct {
+	Field string
+	Index int
+	Start string
+	End   string
+}
+
+func buildFrontmatterTemplate(data map[string]any) (string, []frontmatterMarker) {
+	if len(data) == 0 {
+		return "", nil
+	}
+	markers := []frontmatterMarker{}
+	lines := []string{}
+
+	if summary, ok := data["summary"].(string); ok {
+		start, end := markerPair("SUMMARY", 0)
+		markers = append(markers, frontmatterMarker{Field: "summary", Index: 0, Start: start, End: end})
+		lines = append(lines, fmt.Sprintf("summary: %s%s%s", start, summary, end))
+	}
+
+	if title, ok := data["title"].(string); ok {
+		start, end := markerPair("TITLE", 0)
+		markers = append(markers, frontmatterMarker{Field: "title", Index: 0, Start: start, End: end})
+		lines = append(lines, fmt.Sprintf("title: %s%s%s", start, title, end))
+	}
+
+	if readWhen, ok := data["read_when"].([]any); ok {
+		lines = append(lines, "read_when:")
+		for idx, item := range readWhen {
+			textValue, ok := item.(string)
+			if !ok {
+				lines = append(lines, fmt.Sprintf("  - %v", item))
+				continue
+			}
+			start, end := markerPair("READ_WHEN", idx)
+			markers = append(markers, frontmatterMarker{Field: "read_when", Index: idx, Start: start, End: end})
+			lines = append(lines, fmt.Sprintf("  - %s%s%s", start, textValue, end))
+		}
+	}
+
+	return strings.Join(lines, "\n"), markers
+}
+
+func markerPair(field string, index int) (string, string) {
+	return fmt.Sprintf("[[[FM_%s_%d_START]]]", field, index), fmt.Sprintf("[[[FM_%s_%d_END]]]", field, index)
+}
+
+func applyFrontmatterTranslations(data map[string]any, markers []frontmatterMarker, translatedFront string) error {
+	if len(markers) == 0 {
+		return nil
+	}
+	for _, marker := range markers {
+		value, err := extractMarkerValue(translatedFront, marker.Start, marker.End)
+		if err != nil {
+			return err
+		}
+		value = strings.TrimSpace(value)
+		switch marker.Field {
+		case "summary":
+			data["summary"] = value
+		case "title":
+			data["title"] = value
+		case "read_when":
+			data["read_when"] = setReadWhenValue(data["read_when"], marker.Index, value)
+		}
+	}
+	return nil
+}
+
+func extractMarkerValue(text, start, end string) (string, error) {
+	startIndex := strings.Index(text, start)
+	if startIndex == -1 {
+		return "", fmt.Errorf("missing marker %s", start)
+	}
+	startIndex += len(start)
+	endIndex := strings.Index(text[startIndex:], end)
+	if endIndex == -1 {
+		return "", fmt.Errorf("missing marker %s", end)
+	}
+	endIndex += startIndex
+	return text[startIndex:endIndex], nil
+}
+
+func setReadWhenValue(existing any, index int, value string) []any {
+	readWhen, ok := existing.([]any)
+	if !ok {
+		readWhen = []any{}
+	}
+	for len(readWhen) <= index {
+		readWhen = append(readWhen, "")
+	}
+	readWhen[index] = value
+	return readWhen
+}
+
+func shouldSkipDoc(outputPath string, sourceHash string) (bool, error) {
+	data, err := os.ReadFile(outputPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	frontMatter, _ := splitFrontMatter(string(data))
+	if frontMatter == "" {
+		return false, nil
+	}
+	frontData := map[string]any{}
+	if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
+		return false, nil
+	}
+	storedHash := extractSourceHash(frontData)
+	if storedHash == "" {
+		return false, nil
+	}
+	return strings.EqualFold(storedHash, sourceHash), nil
+}
+
+func extractSourceHash(frontData map[string]any) string {
+	xi, ok := frontData["x-i18n"].(map[string]any)
+	if !ok {
+		return ""
+	}
+	value, ok := xi["source_hash"].(string)
+	if !ok {
+		return ""
+	}
+	return strings.TrimSpace(value)
+}
+
+func resolveDocsPath(docsRoot, filePath string) (string, string, error) {
+	absPath, err := filepath.Abs(filePath)
+	if err != nil {
+		return "", "", err
+	}
+	relPath, err := filepath.Rel(docsRoot, absPath)
+	if err != nil {
+		return "", "", err
+	}
+	if relPath == "." || relPath == "" {
+		return "", "", fmt.Errorf("file %s resolves to docs root %s", absPath, docsRoot)
+	}
+	if filepath.IsAbs(relPath) || relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) {
+		return "", "", fmt.Errorf("file %s not under docs root %s", absPath, docsRoot)
+	}
+	return absPath, relPath, nil
+}
--- a/scripts/docs-i18n/main.go
+++ b/scripts/docs-i18n/main.go
@@ -4,15 +4,38 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
+	"os"
 	"path/filepath"
+	"sync"
+	"time"
 )

+type docJob struct {
+	index int
+	path  string
+	rel   string
+}
+
+type docResult struct {
+	index    int
+	rel      string
+	duration time.Duration
+	skipped  bool
+	err      error
+}
+
 func main() {
 	var (
 		targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)")
 		sourceLang = flag.String("src", "en", "source language")
 		docsRoot   = flag.String("docs", "docs", "docs root")
 		tmPath     = flag.String("tm", "", "translation memory path")
+		mode       = flag.String("mode", "segment", "translation mode (segment|doc)")
+		thinking   = flag.String("thinking", "high", "thinking level (low|high)")
+		overwrite  = flag.Bool("overwrite", false, "overwrite existing translations")
+		maxFiles   = flag.Int("max", 0, "max files to process (0 = all)")
+		parallel   = flag.Int("parallel", 1, "parallel workers for doc mode")
 	)
 	flag.Parse()
 	files := flag.Args()
@@ -35,7 +58,7 @@ func main() {
 		fatal(err)
 	}

-	translator, err := NewPiTranslator(*sourceLang, *targetLang, glossary)
+	translator, err := NewPiTranslator(*sourceLang, *targetLang, glossary, *thinking)
 	if err != nil {
 		fatal(err)
 	}
@@ -46,13 +69,205 @@ func main() {
 		fatal(err)
 	}

-	for _, file := range files {
-		if err := processFile(context.Background(), translator, tm, resolvedDocsRoot, file, *sourceLang, *targetLang); err != nil {
+	ordered, err := orderFiles(resolvedDocsRoot, files)
+	if err != nil {
+		fatal(err)
+	}
+	totalFiles := len(ordered)
+	preSkipped := 0
+	if *mode == "doc" && !*overwrite {
+		filtered, skipped, err := filterDocQueue(resolvedDocsRoot, *targetLang, ordered)
+		if err != nil {
 			fatal(err)
 		}
+		ordered = filtered
+		preSkipped = skipped
+	}
+	if *maxFiles > 0 && *maxFiles < len(ordered) {
+		ordered = ordered[:*maxFiles]
+	}
+
+	log.SetFlags(log.LstdFlags)
+	start := time.Now()
+	processed := 0
+	skipped := 0
+
+	if *parallel < 1 {
+		*parallel = 1
+	}
+
+	log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", *mode, totalFiles, len(ordered), preSkipped, *overwrite, *thinking, *parallel)
+	switch *mode {
+	case "doc":
+		if *parallel > 1 {
+			proc, skip, err := runDocParallel(context.Background(), ordered, resolvedDocsRoot, *sourceLang, *targetLang, *overwrite, *parallel, glossary, *thinking)
+			if err != nil {
+				fatal(err)
+			}
+			processed += proc
+			skipped += skip
+		} else {
+			proc, skip, err := runDocSequential(context.Background(), ordered, translator, resolvedDocsRoot, *sourceLang, *targetLang, *overwrite)
+			if err != nil {
+				fatal(err)
+			}
+			processed += proc
+			skipped += skip
+		}
+	case "segment":
+		if *parallel > 1 {
+			fatal(fmt.Errorf("parallel processing is only supported in doc mode"))
+		}
+		proc, err := runSegmentSequential(context.Background(), ordered, translator, tm, resolvedDocsRoot, *sourceLang, *targetLang)
+		if err != nil {
+			fatal(err)
+		}
+		processed += proc
+	default:
+		fatal(fmt.Errorf("unknown mode: %s", *mode))
 	}

 	if err := tm.Save(); err != nil {
 		fatal(err)
 	}
+	elapsed := time.Since(start).Round(time.Millisecond)
+	log.Printf("docs-i18n: completed processed=%d skipped=%d elapsed=%s", processed, skipped, elapsed)
+}
+
+func runDocSequential(ctx context.Context, ordered []string, translator *PiTranslator, docsRoot, srcLang, tgtLang string, overwrite bool) (int, int, error) {
+	processed := 0
+	skipped := 0
+	for index, file := range ordered {
+		relPath := resolveRelPath(docsRoot, file)
+		log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
+		start := time.Now()
+		skip, err := processFileDoc(ctx, translator, docsRoot, file, srcLang, tgtLang, overwrite)
+		if err != nil {
+			return processed, skipped, err
+		}
+		if skip {
+			skipped++
+			log.Printf("docs-i18n: [%d/%d] skipped %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
+		} else {
+			processed++
+			log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
+		}
+	}
+	return processed, skipped, nil
+}
+
+func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tgtLang string, overwrite bool, parallel int, glossary []GlossaryEntry, thinking string) (int, int, error) {
+	jobs := make(chan docJob)
+	results := make(chan docResult, len(ordered))
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	for worker := 0; worker < parallel; worker++ {
+		wg.Add(1)
+		go func(workerID int) {
+			defer wg.Done()
+			translator, err := NewPiTranslator(srcLang, tgtLang, glossary, thinking)
+			if err != nil {
+				results <- docResult{err: err}
+				return
+			}
+			defer translator.Close()
+			for job := range jobs {
+				if ctx.Err() != nil {
+					return
+				}
+				log.Printf("docs-i18n: [w%d %d/%d] start %s", workerID, job.index, len(ordered), job.rel)
+				start := time.Now()
+				skip, err := processFileDoc(ctx, translator, docsRoot, job.path, srcLang, tgtLang, overwrite)
+				results <- docResult{
+					index:    job.index,
+					rel:      job.rel,
+					duration: time.Since(start),
+					skipped:  skip,
+					err:      err,
+				}
+				if err != nil {
+					cancel()
+					return
+				}
+			}
+		}(worker + 1)
+	}
+
+	go func() {
+		for index, file := range ordered {
+			jobs <- docJob{index: index + 1, path: file, rel: resolveRelPath(docsRoot, file)}
+		}
+		close(jobs)
+	}()
+
+	processed := 0
+	skipped := 0
+	for i := 0; i < len(ordered); i++ {
+		result := <-results
+		if result.err != nil {
+			wg.Wait()
+			return processed, skipped, result.err
+		}
+		if result.skipped {
+			skipped++
+			log.Printf("docs-i18n: [w* %d/%d] skipped %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
+		} else {
+			processed++
+			log.Printf("docs-i18n: [w* %d/%d] done %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
+		}
+	}
+	wg.Wait()
+	return processed, skipped, nil
+}
+
+func runSegmentSequential(ctx context.Context, ordered []string, translator *PiTranslator, tm *TranslationMemory, docsRoot, srcLang, tgtLang string) (int, error) {
+	processed := 0
+	for index, file := range ordered {
+		relPath := resolveRelPath(docsRoot, file)
+		log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
+		start := time.Now()
+		if _, err := processFile(ctx, translator, tm, docsRoot, file, srcLang, tgtLang); err != nil {
+			return processed, err
+		}
+		processed++
+		log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
+	}
+	return processed, nil
+}
+
+func resolveRelPath(docsRoot, file string) string {
+	relPath := file
+	if _, rel, err := resolveDocsPath(docsRoot, file); err == nil {
+		relPath = rel
+	}
+	return relPath
+}
+
+func filterDocQueue(docsRoot, targetLang string, ordered []string) ([]string, int, error) {
+	pending := make([]string, 0, len(ordered))
+	skipped := 0
+	for _, file := range ordered {
+		absPath, relPath, err := resolveDocsPath(docsRoot, file)
+		if err != nil {
+			return nil, skipped, err
+		}
+		content, err := os.ReadFile(absPath)
+		if err != nil {
+			return nil, skipped, err
+		}
+		sourceHash := hashBytes(content)
+		outputPath := filepath.Join(docsRoot, targetLang, relPath)
+		skip, err := shouldSkipDoc(outputPath, sourceHash)
+		if err != nil {
+			return nil, skipped, err
+		}
+		if skip {
+			skipped++
+			continue
+		}
+		pending = append(pending, file)
+	}
+	return pending, skipped, nil
 }
--- a/scripts/docs-i18n/order.go
+++ b/scripts/docs-i18n/order.go
@@ -0,0 +1,37 @@
+package main
+
+import (
+	"path/filepath"
+	"sort"
+)
+
+type orderedFile struct {
+	path string
+	rel  string
+}
+
+func orderFiles(docsRoot string, files []string) ([]string, error) {
+	entries := make([]orderedFile, 0, len(files))
+	for _, file := range files {
+		abs, err := filepath.Abs(file)
+		if err != nil {
+			return nil, err
+		}
+		rel, err := filepath.Rel(docsRoot, abs)
+		if err != nil {
+			rel = abs
+		}
+		entries = append(entries, orderedFile{path: file, rel: rel})
+	}
+	if len(entries) == 0 {
+		return nil, nil
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].rel < entries[j].rel
+	})
+	ordered := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		ordered = append(ordered, entry.path)
+	}
+	return ordered, nil
+}
--- a/scripts/docs-i18n/process.go
+++ b/scripts/docs-i18n/process.go
@@ -11,47 +11,37 @@ import (
 	"gopkg.in/yaml.v3"
 )

-func processFile(ctx context.Context, translator *PiTranslator, tm *TranslationMemory, docsRoot, filePath, srcLang, tgtLang string) error {
-	absPath, err := filepath.Abs(filePath)
+func processFile(ctx context.Context, translator *PiTranslator, tm *TranslationMemory, docsRoot, filePath, srcLang, tgtLang string) (bool, error) {
+	absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
 	if err != nil {
-		return err
-	}
-	relPath, err := filepath.Rel(docsRoot, absPath)
-	if err != nil {
-		return err
-	}
-	if relPath == "." || relPath == "" {
-		return fmt.Errorf("file %s resolves to docs root %s", absPath, docsRoot)
-	}
-	if filepath.IsAbs(relPath) || relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) {
-		return fmt.Errorf("file %s not under docs root %s", absPath, docsRoot)
+		return false, err
 	}

 	content, err := os.ReadFile(absPath)
 	if err != nil {
-		return err
+		return false, err
 	}

 	frontMatter, body := splitFrontMatter(string(content))
 	frontData := map[string]any{}
 	if frontMatter != "" {
 		if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
-			return fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
+			return false, fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
 		}
 	}

 	if err := translateFrontMatter(ctx, translator, tm, frontData, relPath, srcLang, tgtLang); err != nil {
-		return err
+		return false, err
 	}

 	body, err = translateHTMLBlocks(ctx, translator, body, srcLang, tgtLang)
 	if err != nil {
-		return err
+		return false, err
 	}

 	segments, err := extractSegments(body, relPath)
 	if err != nil {
-		return err
+		return false, err
 	}

 	namespace := cacheNamespace()
@@ -64,7 +54,7 @@ func processFile(ctx context.Context, translator *PiTranslator, tm *TranslationM
 		}
 		translated, err := translator.Translate(ctx, seg.Text, srcLang, tgtLang)
 		if err != nil {
-			return fmt.Errorf("translate failed (%s): %w", relPath, err)
+			return false, fmt.Errorf("translate failed (%s): %w", relPath, err)
 		}
 		seg.Translated = translated
 		entry := TMEntry{
@@ -86,16 +76,16 @@ func processFile(ctx context.Context, translator *PiTranslator, tm *TranslationM
 	translatedBody := applyTranslations(body, segments)
 	updatedFront, err := encodeFrontMatter(frontData, relPath, content)
 	if err != nil {
-		return err
+		return false, err
 	}

 	outputPath := filepath.Join(docsRoot, tgtLang, relPath)
 	if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
-		return err
+		return false, err
 	}

 	output := updatedFront + translatedBody
-	return os.WriteFile(outputPath, []byte(output), 0o644)
+	return false, os.WriteFile(outputPath, []byte(output), 0o644)
 }

 func splitFrontMatter(content string) (string, string) {
@@ -125,8 +115,8 @@ func splitFrontMatter(content string) (string, string) {
 }

 func encodeFrontMatter(frontData map[string]any, relPath string, source []byte) (string, error) {
-	if len(frontData) == 0 {
-		return "", nil
+	if frontData == nil {
+		frontData = map[string]any{}
 	}
 	frontData["x-i18n"] = map[string]any{
 		"source_path":  relPath,
@@ -154,6 +144,13 @@ func translateFrontMatter(ctx context.Context, translator *PiTranslator, tm *Tra
 		}
 		data["summary"] = translated
 	}
+	if title, ok := data["title"].(string); ok {
+		translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:title", title, srcLang, tgtLang)
+		if err != nil {
+			return err
+		}
+		data["title"] = translated
+	}
 	if readWhen, ok := data["read_when"].([]any); ok {
 		translated := make([]any, 0, len(readWhen))
 		for idx, item := range readWhen {
--- a/scripts/docs-i18n/tm.go
+++ b/scripts/docs-i18n/tm.go
@@ -52,7 +52,7 @@ func LoadTranslationMemory(path string) (*TranslationMemory, error) {
 				if err := json.Unmarshal([]byte(trimmed), &entry); err != nil {
 					return nil, fmt.Errorf("translation memory decode failed: %w", err)
 				}
-				if entry.CacheKey != "" {
+				if entry.CacheKey != "" && strings.TrimSpace(entry.Translated) != "" {
 					tm.entries[entry.CacheKey] = entry
 				}
 			}
@@ -69,7 +69,13 @@ func LoadTranslationMemory(path string) (*TranslationMemory, error) {

 func (tm *TranslationMemory) Get(cacheKey string) (TMEntry, bool) {
 	entry, ok := tm.entries[cacheKey]
-	return entry, ok
+	if !ok {
+		return TMEntry{}, false
+	}
+	if strings.TrimSpace(entry.Translated) == "" {
+		return TMEntry{}, false
+	}
+	return entry, true
 }

 func (tm *TranslationMemory) Put(entry TMEntry) {
--- a/scripts/docs-i18n/translator.go
+++ b/scripts/docs-i18n/translator.go
@@ -2,25 +2,35 @@ package main

 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
+	"time"

 	pi "github.com/joshp123/pi-golang"
 )

+const (
+	translateMaxAttempts = 3
+	translateBaseDelay   = 15 * time.Second
+)
+
+var errEmptyTranslation = errors.New("empty translation")
+
 type PiTranslator struct {
 	client *pi.OneShotClient
 }

-func NewPiTranslator(srcLang, tgtLang string, glossary []GlossaryEntry) (*PiTranslator, error) {
+func NewPiTranslator(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (*PiTranslator, error) {
 	options := pi.DefaultOneShotOptions()
 	options.AppName = "openclaw-docs-i18n"
+	options.WorkDir = "/tmp"
 	options.Mode = pi.ModeDragons
 	options.Dragons = pi.DragonsOptions{
 		Provider: "anthropic",
 		Model:    modelVersion,
-		Thinking: "high",
+		Thinking: normalizeThinking(thinking),
 	}
 	options.SystemPrompt = translationPrompt(srcLang, tgtLang, glossary)
 	client, err := pi.StartOneShot(options)
@@ -31,6 +41,14 @@ func NewPiTranslator(srcLang, tgtLang string, glossary []GlossaryEntry) (*PiTran
 }

 func (t *PiTranslator) Translate(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
+	return t.translate(ctx, text, t.translateMasked)
+}
+
+func (t *PiTranslator) TranslateRaw(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
+	return t.translate(ctx, text, t.translateRaw)
+}
+
+func (t *PiTranslator) translate(ctx context.Context, text string, run func(context.Context, string) (string, error)) (string, error) {
 	if t.client == nil {
 		return "", errors.New("pi client unavailable")
 	}
@@ -38,20 +56,87 @@ func (t *PiTranslator) Translate(ctx context.Context, text, srcLang, tgtLang str
 	if core == "" {
 		return text, nil
 	}
+	translated, err := t.translateWithRetry(ctx, func(ctx context.Context) (string, error) {
+		return run(ctx, core)
+	})
+	if err != nil {
+		return "", err
+	}
+	return prefix + translated + suffix, nil
+}
+
+func (t *PiTranslator) translateWithRetry(ctx context.Context, run func(context.Context) (string, error)) (string, error) {
+	var lastErr error
+	for attempt := 0; attempt < translateMaxAttempts; attempt++ {
+		translated, err := run(ctx)
+		if err == nil {
+			return translated, nil
+		}
+		if !isRetryableTranslateError(err) {
+			return "", err
+		}
+		lastErr = err
+		if attempt+1 < translateMaxAttempts {
+			delay := translateBaseDelay * time.Duration(attempt+1)
+			if err := sleepWithContext(ctx, delay); err != nil {
+				return "", err
+			}
+		}
+	}
+	return "", lastErr
+}
+
+func (t *PiTranslator) translateMasked(ctx context.Context, core string) (string, error) {
 	state := NewPlaceholderState(core)
 	placeholders := make([]string, 0, 8)
 	mapping := map[string]string{}
 	masked := maskMarkdown(core, state.Next, &placeholders, mapping)
-	res, err := t.client.Run(ctx, masked)
+	resText, err := runPrompt(ctx, t.client, masked)
 	if err != nil {
 		return "", err
 	}
-	translated := strings.TrimSpace(res.Text)
+	translated := strings.TrimSpace(resText)
+	if translated == "" {
+		return "", errEmptyTranslation
+	}
 	if err := validatePlaceholders(translated, placeholders); err != nil {
 		return "", err
 	}
-	translated = unmaskMarkdown(translated, placeholders, mapping)
-	return prefix + translated + suffix, nil
+	return unmaskMarkdown(translated, placeholders, mapping), nil
+}
+
+func (t *PiTranslator) translateRaw(ctx context.Context, core string) (string, error) {
+	resText, err := runPrompt(ctx, t.client, core)
+	if err != nil {
+		return "", err
+	}
+	translated := strings.TrimSpace(resText)
+	if translated == "" {
+		return "", errEmptyTranslation
+	}
+	return translated, nil
+}
+
+func isRetryableTranslateError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, errEmptyTranslation) {
+		return true
+	}
+	message := strings.ToLower(err.Error())
+	return strings.Contains(message, "placeholder missing") || strings.Contains(message, "rate limit") || strings.Contains(message, "429")
+}
+
+func sleepWithContext(ctx context.Context, delay time.Duration) error {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
 }

 func (t *PiTranslator) Close() {
@@ -60,6 +145,98 @@ func (t *PiTranslator) Close() {
 	}
 }

+type agentEndPayload struct {
+	Messages []agentMessage `json:"messages"`
+}
+
+type agentMessage struct {
+	Role         string          `json:"role"`
+	Content      json.RawMessage `json:"content"`
+	StopReason   string          `json:"stopReason,omitempty"`
+	ErrorMessage string          `json:"errorMessage,omitempty"`
+}
+
+type contentBlock struct {
+	Type string `json:"type"`
+	Text string `json:"text,omitempty"`
+}
+
+func runPrompt(ctx context.Context, client *pi.OneShotClient, message string) (string, error) {
+	events, cancel := client.Subscribe(256)
+	defer cancel()
+
+	if err := client.Prompt(ctx, message); err != nil {
+		return "", err
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		case event, ok := <-events:
+			if !ok {
+				return "", errors.New("event stream closed")
+			}
+			if event.Type == "agent_end" {
+				return extractTranslationResult(event.Raw)
+			}
+		}
+	}
+}
+
+func extractTranslationResult(raw json.RawMessage) (string, error) {
+	var payload agentEndPayload
+	if err := json.Unmarshal(raw, &payload); err != nil {
+		return "", err
+	}
+	for index := len(payload.Messages) - 1; index >= 0; index-- {
+		message := payload.Messages[index]
+		if message.Role != "assistant" {
+			continue
+		}
+		if message.ErrorMessage != "" || strings.EqualFold(message.StopReason, "error") {
+			msg := strings.TrimSpace(message.ErrorMessage)
+			if msg == "" {
+				msg = "unknown error"
+			}
+			return "", fmt.Errorf("pi error: %s", msg)
+		}
+		text, err := extractContentText(message.Content)
+		if err != nil {
+			return "", err
+		}
+		return text, nil
+	}
+	return "", errors.New("assistant message not found")
+}
+
+func extractContentText(content json.RawMessage) (string, error) {
+	trimmed := strings.TrimSpace(string(content))
+	if trimmed == "" {
+		return "", nil
+	}
+	if strings.HasPrefix(trimmed, "\"") {
+		var text string
+		if err := json.Unmarshal(content, &text); err != nil {
+			return "", err
+		}
+		return text, nil
+	}
+
+	var blocks []contentBlock
+	if err := json.Unmarshal(content, &blocks); err != nil {
+		return "", err
+	}
+
+	var parts []string
+	for _, block := range blocks {
+		if block.Type == "text" && block.Text != "" {
+			parts = append(parts, block.Text)
+		}
+	}
+	return strings.Join(parts, ""), nil
+}
+
 func translationPrompt(srcLang, tgtLang string, glossary []GlossaryEntry) string {
 	srcLabel := srcLang
 	tgtLabel := tgtLang
@@ -75,12 +252,22 @@ Translate from %s to %s.

 Rules:
 - Output ONLY the translated text. No preamble, no questions, no commentary.
+- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
+- All prose must be Chinese. If any English sentence remains outside code/URLs/product names, it is wrong.
+- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
+- Translate only the contents inside those tags.
+- Preserve YAML structure inside <frontmatter>; translate only values.
+- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
+- Translate headings/labels like "Exit codes" and "Optional scripts".
 - Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
+- Preserve HTML tags and attributes exactly.
 - Do not translate code spans/blocks, config keys, CLI flags, or env vars.
 - Do not alter URLs or anchors.
 - Preserve placeholders exactly: __OC_I18N_####__.
- Use neutral technical Chinese; avoid slang or jokes.
+- Do not remove, reorder, or summarize content.
+- Use fluent, idiomatic technical Chinese; avoid slang or jokes.
 - Keep product names in English: OpenClaw, Gateway, Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
+- Never output an empty response; if unsure, return the source text unchanged.

 %s

@@ -102,3 +289,12 @@ func buildGlossaryPrompt(glossary []GlossaryEntry) string {
 	}
 	return strings.Join(lines, "\n")
 }
+
+func normalizeThinking(value string) string {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case "low", "high":
+		return strings.ToLower(strings.TrimSpace(value))
+	default:
+		return "high"
+	}
+}
--- a/scripts/docs-i18n/util.go
+++ b/scripts/docs-i18n/util.go
@@ -10,7 +10,7 @@ import (
 )

 const (
-	workflowVersion = 9
+	workflowVersion = 15
 	providerName    = "pi"
 	modelVersion    = "claude-opus-4-5"
 )