Skip to content

Commit

Permalink
✨ feat: I18n support concurrency query
Browse files Browse the repository at this point in the history
  • Loading branch information
@canisminor1990
canisminor1990 committed Nov 22, 2023
1 parent bb00187 commit f0eafc1
Show file tree
Hide file tree
Showing 7 changed files with 67 additions and 50 deletions.
1 change: 1 addition & 0 deletions packages/lobe-i18n/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ This project provides some additional configuration items set with environment v
| reference | No | `string` | - | Provide some context for more accurate translations |
| splitToken | No | `number` | - | Split the localized JSON file by tokens, automatically calculated by default |
| temperature | No | `number` | `0` | Sampling temperature to use |
| concurrency | No | `number` | `5` | Number of concurrently pending promises returned |
| experimental | No | `experimental` | | Experimental features, see below |
| markdown | No | `markdown` | | See `markdown` configuration below |

Expand Down
21 changes: 11 additions & 10 deletions packages/lobe-i18n/README.zh-CN.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ $ lobe-i18n md
| reference | 否 | `string` | - | 提供一些上下文以获得更准确的翻译 |
| splitToken | 否 | `number` | - | 按令牌分割本地化 JSON 文件，默认自动计算 |
| temperature | 否 | `number` | `0` | 使用的采样温度 |
| concurrency | 否 | `number` | `5` | 同时并发的队列请求数量 |
| experimental | 否 | `experimental` | | 实验性功能，见下文 |
| markdown | 否 | `markdown` | | 见 `markdown` 配置说明 |

Expand Down Expand Up @@ -285,16 +286,16 @@ $ lobe-i18n

## 📝 Markdown 配置

| 属性名称 | 必填 | 类型 | 默认值 | 描述 |
| ---------------- | ---- | ------------------------- | ---------------------------- | ----------------------------------------- |
| entry | 是 | `string[]` | - | 入口文件或文件夹，支持 `glob` |
| entryLocale | 否 | `string` | 同父级 | 作为翻译参考的语言 |
| entryExtension | 否 | `string` | `.md` | 入口文件扩展名 |
| exclude | 否 | `string[]` | - | 需要过滤的文件，支持 `glob` |
| outputLocales | 否 | `string[]` | 同父级 | 需要进行翻译的所有语言 |
| outputExtensions | 否 | `function` | `(locale) => '.{locale}.md'` | 输出文件的扩展名生成 |
| mode | 否 | `string``mdast``function` | `string` | 翻译的模式选择，解释见下文 |
| translateCode | 否 | `boolean` | `false` | 在 `mdast` 下是否翻译代码块，其他模式无效 |
| 属性名称 | 必填 | 类型 | 默认值 | 描述 |
| ---------------- | ---- | --------------------------- | ---------------------------- | ----------------------------------------- |
| entry | 是 | `string[]` | - | 入口文件或文件夹，支持 `glob` |
| entryLocale | 否 | `string` | 同父级 | 作为翻译参考的语言 |
| entryExtension | 否 | `string` | `.md` | 入口文件扩展名 |
| exclude | 否 | `string[]` | - | 需要过滤的文件，支持 `glob` |
| outputLocales | 否 | `string[]` | 同父级 | 需要进行翻译的所有语言 |
| outputExtensions | 否 | `function` | `(locale) => '.{locale}.md'` | 输出文件的扩展名生成 |
| mode | 否 | `string`,`mdast`,`function` | `string` | 翻译的模式选择，解释见下文 |
| translateCode | 否 | `boolean` | `false` | 在 `mdast` 下是否翻译代码块，其他模式无效 |

#### `outputExtensions`

Expand Down
1 change: 1 addition & 0 deletions packages/lobe-i18n/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
"langchain": "latest",
"lodash-es": "^4",
"multimatch": "^7",
"p-map": "^6",
"pangu": "^4",
"react": "^18",
"remark-frontmatter": "^4",
Expand Down
73 changes: 39 additions & 34 deletions packages/lobe-i18n/src/core/I18n.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { merge } from 'lodash-es';
import pMap from 'p-map';

import { TranslateMarkdown } from '@/core/TranslateMarkdown';
import { LocaleObj } from '@/types';
Expand Down Expand Up @@ -81,30 +82,32 @@ export class I18n {
this.maxStep = splitString.length;
this.step = 0;

const translatedSplitString: string[] = [];

onProgress?.({
isLoading: true,
maxStep: this.maxStep,
progress: 0,
step: 0,
});

for (const text of splitString) {
onProgress?.({
isLoading: this.step < this.maxStep,
maxStep: this.maxStep,
progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100,
step: this.step,
});
const result = await this.translateLocaleService.runByString({
from,
text,
to,
});
if (result) translatedSplitString.push(result);
if (this.step < this.maxStep) this.step++;
}
const translatedSplitString: string[] = await pMap(
splitString,
async (text) => {
onProgress?.({
isLoading: this.step < this.maxStep,
maxStep: this.maxStep,
progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100,
step: this.step,
});
const result = await this.translateLocaleService.runByString({
from,
text,
to,
});
if (this.step < this.maxStep) this.step++;
return result;
},
{ concurrency: this.config?.concurrency },
);

onProgress?.({
isLoading: false,
Expand Down Expand Up @@ -150,30 +153,32 @@ export class I18n {
this.maxStep = splitJson.length;
this.step = 0;

const translatedSplitJson: LocaleObj[] = [];

onProgress?.({
isLoading: true,
maxStep: this.maxStep,
progress: 0,
step: 0,
});

for (const json of splitJson) {
onProgress?.({
isLoading: this.step < this.maxStep,
maxStep: this.maxStep,
progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100,
step: this.step,
});
const result = await this.translateLocaleService.runByJson({
from,
json,
to,
});
if (result) translatedSplitJson.push(result);
if (this.step < this.maxStep) this.step++;
}
const translatedSplitJson: LocaleObj[] = await pMap(
splitJson,
async (json) => {
onProgress?.({
isLoading: this.step < this.maxStep,
maxStep: this.maxStep,
progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100,
step: this.step,
});
const result = await this.translateLocaleService.runByJson({
from,
json,
to,
});
if (this.step < this.maxStep) this.step++;
return result;
},
{ concurrency: this.config?.concurrency },
);

onProgress?.({
isLoading: false,
Expand Down
11 changes: 10 additions & 1 deletion packages/lobe-i18n/src/store/initialState.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
import { I18nConfig } from '@/types/config';
import { I18nConfig, MarkdownModeType } from '@/types/config';
import { LanguageModel } from '@/types/models';
import { getDefaultExtension } from '@/utils/getDefaultExtension';

export const DEFAULT_CONFIG: Partial<I18nConfig> = {
concurrency: 5,
markdown: {
entry: [],
entryExtension: '.md',
mode: MarkdownModeType.STRING,
outputExtensions: getDefaultExtension,
},
modelName: LanguageModel.GPT3_5,
temperature: 0,
};
6 changes: 1 addition & 5 deletions packages/lobe-i18n/src/store/selectors.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@ import dotenv from 'dotenv';
import { merge } from 'lodash-es';

import { DEFAULT_CONFIG } from '@/store/initialState';
import { Config, ConfigKeys, I18nConfig, MarkdownConfig, MarkdownModeType } from '@/types/config';
import { Config, ConfigKeys, I18nConfig, MarkdownConfig } from '@/types/config';
import { checkOptionKeys } from '@/utils/checkOptionKeys';
import { getDefaultExtension } from '@/utils/getDefaultExtension';

import { config, explorer, schema } from './config';

Expand Down Expand Up @@ -42,10 +41,7 @@ const getMarkdownConfigFile = (): MarkdownConfig => {
}

const markdownConfig = merge(config?.markdown || {}, {
entryExtension: config?.markdown?.entryExtension || '.md',
entryLocale: config?.markdown?.entryLocale || config.entryLocale,
mode: config?.markdown?.mode || MarkdownModeType.STRING,
outputExtensions: config?.markdown?.outputExtensions || getDefaultExtension,
outputLocales: config?.markdown?.outputLocales || config.outputLocales,
});

Expand Down
4 changes: 4 additions & 0 deletions packages/lobe-i18n/src/types/config.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import { LanguageModel } from './models';

export interface I18nConfigLocale {
/**
* @description Number of concurrently pending promises returned
*/
concurrency?: number;
/**
* @description The entry file or folder
*/
Expand Down

0 comments on commit f0eafc1

Please sign in to comment.