✨ feat: I18n add token info output

packages/lobe-i18n/src/commands/TranslateLocale/index.tsx

@@ -44,6 +44,12 @@ class TranslateLocale {
   }
 
   async runQuery() {
+    consola.info(
+      `Current model setting: ${chalk.cyan(this.config.modelName)} (temperature: ${chalk.cyan(
+        this.config.temperature,
+      )}) ${this.config.experimental?.jsonMode ? chalk.red(' [JSON Mode]') : ''}}`,
+    );
+    let totalTokenUsage = 0;
     for (const item of this.query) {
       const props = {
         filename: item.filename,
@@ -64,11 +70,16 @@ class TranslateLocale {
         },
       });
       clear();
-      if (data && Object.keys(data).length > 0) {
-        writeJSON(item.filename, data);
-        consola.success(chalk.yellow(relative(this.config.output, item.filename)));
+      const outputPath = relative('.', item.filename);
+      if (data?.result && Object.keys(data.result).length > 0) {
+        writeJSON(item.filename, data.result);
+        totalTokenUsage += data.tokenUsage;
+        consola.success(chalk.yellow(outputPath), chalk.gray(`[Token usage: ${data.tokenUsage}]`));
+      } else {
+        consola.warn('No translation result was found:', chalk.yellow(outputPath));
       }
     }
+    if (totalTokenUsage > 0) consola.info('Total token usage:', chalk.cyan(totalTokenUsage));
   }
 
   genFolderQuery() {

packages/lobe-i18n/src/commands/TranslateMarkdown/index.tsx

@@ -60,6 +60,12 @@ class TranslateMarkdown {
   }
 
   async runQuery() {
+    consola.info(
+      `Current model setting: ${chalk.cyan(this.config.modelName)} (temperature: ${chalk.cyan(
+        this.config.temperature,
+      )}) ${this.config.experimental?.jsonMode ? chalk.red(' [JSON Mode]') : ''}}`,
+    );
+    let totalTokenUsage = 0;
     for (const item of this.query) {
       const props = {
         filename: item.filename,
@@ -81,11 +87,16 @@ class TranslateMarkdown {
         },
       });
       clear();
-      if (data && Object.keys(data).length > 0) {
-        writeMarkdown(item.filename, data);
-        consola.success(chalk.yellow(relative('.', item.filename)));
+      const outputPath = relative('.', item.filename);
+      if (data?.result && Object.keys(data.result).length > 0) {
+        writeMarkdown(item.filename, data.result);
+        totalTokenUsage += data.tokenUsage;
+        consola.success(chalk.yellow(outputPath), chalk.gray(`[Token usage: ${data.tokenUsage}]`));
+      } else {
+        consola.warn('No translation result was found:', chalk.yellow(outputPath));
       }
     }
+    if (totalTokenUsage > 0) consola.info('Total token usage:', chalk.cyan(totalTokenUsage));
   }
 
   genFilesQuery(files: string[], skipLog?: boolean) {

packages/lobe-i18n/src/components/Progress.tsx

@@ -13,7 +13,7 @@ interface ProgressProps extends onProgressProps {
 }
 
 const Progress = memo<ProgressProps>(
-  ({ hide, filename, to, from, progress, maxStep, step, isLoading }) => {
+  ({ hide, filename, to, from, progress, maxStep, step, isLoading, needToken }) => {
     const theme = useTheme();
 
     if (hide) return null;
@@ -32,6 +32,7 @@ const Progress = memo<ProgressProps>(
           <Text bold color={theme.colorInfo}>
             {to}
           </Text>
+          <Text color={theme.colorTextDescription}>{` [Tokens: ${needToken}]`}</Text>
         </Text>
         {isLoading ? (
           <Box>

packages/lobe-i18n/src/core/I18n.ts

@@ -4,6 +4,7 @@ import pMap from 'p-map';
 import { TranslateMarkdown } from '@/core/TranslateMarkdown';
 import { LocaleObj } from '@/types';
 import { I18nConfig, MarkdownModeType } from '@/types/config';
+import { calcToken } from '@/utils/calcToken';
 import { mergeJsonFromChunks } from '@/utils/mergeJsonFromChunks';
 import { splitJsonToChunks } from '@/utils/splitJsonToChunks';
 
@@ -18,6 +19,7 @@ export interface I18nOptions {
 export interface onProgressProps {
   isLoading: boolean;
   maxStep: number;
+  needToken?: number;
   progress: number;
   step: number;
 }
@@ -55,7 +57,13 @@ export class I18n {
     this.translateMarkdownService = new TranslateMarkdown(config);
   }
 
-  async translateMarkdown(options: I18nMarkdownTranslateOptions): Promise<string | undefined> {
+  async translateMarkdown(options: I18nMarkdownTranslateOptions): Promise<
+    | {
+        result: string;
+        tokenUsage: number;
+      }
+    | undefined
+  > {
     return options.mode === MarkdownModeType.STRING
       ? this.translateMarkdownByString(options)
       : this.translateMarkdownByMdast(options);
@@ -66,7 +74,7 @@ export class I18n {
     to,
     onProgress,
     from,
-  }: I18nMarkdownTranslateOptions): Promise<string | undefined> {
+  }: I18nMarkdownTranslateOptions): Promise<{ result: string; tokenUsage: number } | undefined> {
     const prompt = await this.translateLocaleService.promptString.formatMessages({
       from,
       text: '',
@@ -82,9 +90,14 @@ export class I18n {
 
     if (splitString.length === 0) return;
 
+    const needToken =
+      splitString.length * calcToken(JSON.stringify(prompt)) +
+      calcToken(JSON.stringify(splitString));
+
     onProgress?.({
       isLoading: true,
       maxStep: this.maxStep,
+      needToken,
       progress: 0,
       step: 0,
     });
@@ -95,6 +108,7 @@ export class I18n {
         onProgress?.({
           isLoading: this.step < this.maxStep,
           maxStep: this.maxStep,
+          needToken,
           progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100,
           step: this.step,
         });
@@ -112,17 +126,26 @@ export class I18n {
     onProgress?.({
       isLoading: false,
       maxStep: this.maxStep,
+      needToken,
       progress: 100,
       step: this.maxStep,
     });
 
-    return this.translateMarkdownService.genMarkdownByString(translatedSplitString);
+    const result = await this.translateMarkdownService.genMarkdownByString(translatedSplitString);
+
+    return {
+      result,
+      tokenUsage: needToken + calcToken(JSON.stringify(translatedSplitString)),
+    };
   }
 
-  async translateMarkdownByMdast({
-    md,
-    ...rest
-  }: I18nMarkdownTranslateOptions): Promise<string | undefined> {
+  async translateMarkdownByMdast({ md, ...rest }: I18nMarkdownTranslateOptions): Promise<
+    | {
+        result: string;
+        tokenUsage: number;
+      }
+    | undefined
+  > {
     const target = await this.translateMarkdownService.genTarget(md);
 
     const translatedTarget = await this.translate({
@@ -131,16 +154,25 @@ export class I18n {
       target: {},
     });
 
-    return this.translateMarkdownService.genMarkdownByMdast(translatedTarget);
+    if (!translatedTarget?.result) return;
+
+    const result = await this.translateMarkdownService.genMarkdownByMdast(translatedTarget);
+
+    if (!result) return;
+
+    return {
+      result,
+      tokenUsage: translatedTarget.tokenUsage,
+    };
   }
 
-  async translate({
-    entry,
-    target,
-    to,
-    onProgress,
-    from,
-  }: I18nTranslateOptions): Promise<LocaleObj | undefined> {
+  async translate({ entry, target, to, onProgress, from }: I18nTranslateOptions): Promise<
+    | {
+        result: LocaleObj;
+        tokenUsage: number;
+      }
+    | undefined
+  > {
     const prompt = await this.translateLocaleService.promptJson.formatMessages({
       from,
       json: {},
@@ -153,9 +185,13 @@ export class I18n {
 
     if (splitJson.length === 0) return;
 
+    const needToken =
+      splitJson.length * calcToken(JSON.stringify(prompt)) + calcToken(JSON.stringify(splitJson));
+
     onProgress?.({
       isLoading: true,
       maxStep: this.maxStep,
+      needToken,
       progress: 0,
       step: 0,
     });
@@ -166,6 +202,7 @@ export class I18n {
         onProgress?.({
           isLoading: this.step < this.maxStep,
           maxStep: this.maxStep,
+          needToken,
           progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100,
           step: this.step,
         });
@@ -183,10 +220,16 @@ export class I18n {
     onProgress?.({
       isLoading: false,
       maxStep: this.maxStep,
+      needToken,
       progress: 100,
       step: this.maxStep,
     });
 
-    return merge(target, mergeJsonFromChunks(translatedSplitJson));
+    const result = await merge(target, mergeJsonFromChunks(translatedSplitJson));
+
+    return {
+      result,
+      tokenUsage: needToken + calcToken(JSON.stringify(translatedSplitJson)),
+    };
   }
 }

packages/lobe-i18n/src/prompts/translate.ts

@@ -1,7 +1,7 @@
 import { ChatPromptTemplate } from 'langchain/prompts';
 
 const DEFAULT_REFERENCE =
-  'You can adjust the tone and style, taking into account the cultural connotations and regional differences of certain words. As a translator, you need to translate the original text into a translation that meets the standards of accuracy and elegance. "Accuracy" means being faithful to the content and intent of the original text; "elegance" means that the translation should be clear and easy to understand, with a clear expression; "elegance" pursues the cultural aesthetics of the translation and the beauty of the language. The goal is to create a translation that is both faithful to the spirit of the original work and conforms to the target language and reader\'s aesthetic preferences.';
+  'You can adjust the tone and style, taking into account the cultural connotations and regional differences of certain words. As a translator, you need to translate the original text into a translation that meets the standards of accuracy and elegance.';
 export const promptJsonTranslate = (reference: string = DEFAULT_REFERENCE) => {
   return ChatPromptTemplate.fromPromptMessages<{
     from: string;