test: remove fixtures

.env.example

@@ -19,3 +19,6 @@ TENCENT_SECRETKEY=
 # http://ai.youdao.com/gw.s
 YOUDAO_APPKEY=
 YOUDAO_KEY=
+
+# moji dict
+MOJI_ID=

package.json

@@ -14,6 +14,7 @@
     "zip": "yarn neutrino-webextension-zip",
     "commit": "git-cz",
     "release": "standard-version",
+    "fixtures": "node scripts/fixtures.js",
     "pdf": "node scripts/pdf.js"
   },
   "husky": {
@@ -116,6 +117,7 @@
     "axios-mock-adapter": "^1.17.0",
     "babel-plugin-import": "^1.12.1",
     "clean-css-loader": "^2.0.0",
+    "cli-progress": "^3.5.0",
     "commitizen": "^3.1.1",
     "cz-conventional-changelog": "^2.1.0",
     "dotenv-webpack": "^1.7.0",
@@ -130,6 +132,7 @@
     "eslint-plugin-react": "^7.14.2",
     "eslint-plugin-standard": "^4.0.0",
     "faker": "^4.1.0",
+    "fast-glob": "^3.1.1",
     "form-data": "^2.5.1",
     "husky": "^3.0.0",
     "jest": "^24.9.0",
@@ -142,6 +145,7 @@
     "postcss-loader": "^3.0.0",
     "prettier": "^1.18.2",
     "raf": "^3.4.1",
+    "random-mua": "^0.5.0",
     "raw-loader": "^3.1.0",
     "react-docgen-typescript-loader": "^3.1.0",
     "sass-loader": "^7.1.0",

scripts/fixtures.js

@@ -0,0 +1,158 @@
+const path = require('path')
+const fs = require('fs-extra')
+const axios = require('axios')
+const fglob = require('fast-glob')
+const cliProgress = require('cli-progress')
+const randomMua = require('random-mua')
+const argv = require('yargs').argv
+const env = require('dotenv').config({
+  path: path.join(__dirname, '../.env')
+}).parsed
+
+// download fixtures
+// --delete remove all fixtures
+
+if (argv.delete) {
+  deletion()
+} else {
+  main().catch(console.log)
+}
+
+async function main() {
+  const proxyConfig = env.PROXY_HOST
+    ? {
+        proxy: {
+          host: env.PROXY_HOST,
+          port: env.PROXY_PORT
+        }
+      }
+    : {}
+  if (env.PROXY_HOST) {
+    console.log(`with proxy:${env.PROXY_HOST}:${env.PROXY_PORT}`)
+  }
+
+  const progressBars = new cliProgress.MultiBar({
+    format: ' {bar} | "{file}" | {value}/{total} | {status}',
+    hideCursor: true,
+    barCompleteChar: '\u2588',
+    barIncompleteChar: '\u2591',
+    clearOnComplete: false,
+    stopOnComplete: true
+  })
+
+  const errors = []
+
+  const fixturesPath = await fglob(['**/fixtures.js'], {
+    cwd: path.join(__dirname, '../test'),
+    absolute: true,
+    onlyFiles: true
+  })
+
+  await Promise.all(fixturesPath.map(fetchDictFixtures))
+
+  if (errors.length > 0) {
+    console.log('\nErrors:\n\n', errors.map(([name]) => name).join('\n'))
+    await fs.outputFile(
+      path.join(__dirname, 'fixtures.log'),
+      errors.map(([name, error]) => name + '\n' + error).join('\n\n')
+    )
+  }
+
+  async function fetchDictFixtures(fixturePath) {
+    const fixture = require(fixturePath)
+
+    const fetched = []
+
+    for (const [filename, fetchUrl] of fixture.files) {
+      const destPath = fixturePath.replace(
+        /fixtures.js$/,
+        `response/${filename}`
+      )
+      const stat = await fs.stat(destPath).catch(() => null)
+      if (stat && stat.isFile()) {
+        fetched.push(await fs.readFile(destPath, 'utf8'))
+        continue
+      }
+
+      const dictname = /[\\/]+([^\\/]+)[\\/]+fixtures.js$/.exec(fixturePath)[1]
+
+      const pgBar = progressBars.create(100, 0, {
+        file: `${dictname}/${filename}`,
+        status: 'downloading'
+      })
+
+      try {
+        var customConfig =
+          typeof fetchUrl === 'string'
+            ? {
+                url: fetchUrl
+              }
+            : fetchUrl(fetched)
+      } catch (e) {
+        pgBar.update(null, { status: 'parse error' })
+        pgBar.stop()
+        continue
+      }
+
+      const { origin, host } = new URL(customConfig.url)
+
+      if (!customConfig) {
+        pgBar.update(null, { status: 'empty config' })
+        pgBar.stop()
+        continue
+      }
+
+      const axiosConfig = {
+        transformResponse: [data => data],
+        ...proxyConfig,
+        ...customConfig,
+        headers: {
+          'user-agent': randomMua(),
+          ...(customConfig.headers || {})
+        }
+      }
+
+      try {
+        const { data: result } = await axios(axiosConfig).catch(e =>
+          e.response && e.response.status === 404 && e.response.data
+            ? e.response
+            : axios({
+                ...axiosConfig,
+                headers: {
+                  ...axiosConfig.headers,
+                  origin,
+                  host,
+                  referer: origin
+                }
+              })
+        )
+
+        fetched.push(result)
+        await fs.outputFile(
+          fixturePath.replace(/fixtures.js$/, `response/${filename}`),
+          result
+        )
+
+        pgBar.update(100, { status: 'success' })
+        pgBar.stop()
+      } catch (e) {
+        errors.push([`${dictname}/${filename}`, e])
+
+        pgBar.update(null, { status: 'failed' })
+        pgBar.stop()
+      }
+    }
+  }
+}
+
+async function deletion() {
+  const fixturesPath = await fglob(['**/response'], {
+    cwd: path.join(__dirname, '../test'),
+    absolute: true,
+    onlyDirectories: true
+  })
+
+  fixturesPath.forEach(fixturePath => {
+    fs.remove(fixturePath)
+  })
+}

src/components/dictionaries/mojidict/engine.ts

@@ -6,8 +6,6 @@ import {
 } from '../helpers'
 import axios, { AxiosResponse } from 'axios'
 
-const APPLICATION_ID = 'E62VyFVLMiW7kvbtVq3p'
-
 export const getSrcPage: GetSrcPageFunction = async text => {
   const suggests = await getSuggests(text).catch(() => null)
   if (suggests) {
@@ -115,7 +113,7 @@ export const search: SearchFunction<MojidictResult> = async (
     },
     data: JSON.stringify({
       wordId,
-      _ApplicationId: APPLICATION_ID,
+      _ApplicationId: process.env.MOJI_ID,
       _ClientVersion: 'js2.7.1',
       _InstallationId: getInstallationId()
     })
@@ -181,7 +179,7 @@ async function getSuggests(text: string): Promise<SuggestsResult> {
         langEnv: 'zh-CN_ja',
         needWords: true,
         searchText: text,
-        _ApplicationId: APPLICATION_ID,
+        _ApplicationId: process.env.MOJI_ID,
         _ClientVersion: 'js2.7.1',
         _InstallationId: getInstallationId()
       })
@@ -204,7 +202,7 @@ async function getTTS(text: string, wordId: string): Promise<string> {
       data: JSON.stringify({
         identity: wordId,
         text,
-        _ApplicationId: APPLICATION_ID,
+        _ApplicationId: process.env.MOJI_ID,
         _ClientVersion: 'js2.7.1',
         _InstallationId: getInstallationId()
       })

test/specs/components/dictionaries/bing/fixtures.js

@@ -0,0 +1,12 @@
+module.exports = {
+  files: [
+    ['lex.html', 'https://cn.bing.com/dict/search?q=love'],
+    [
+      'machine.html',
+      `https://cn.bing.com/dict/search?q=${encodeURIComponent(
+        'lose yourself in the dark'
+      )}`
+    ],
+    ['related.html', 'https://cn.bing.com/dict/search?q=lovxx']
+  ]
+}

test/specs/components/dictionaries/cambridge/fixtures.js

@@ -0,0 +1,16 @@
+module.exports = {
+  files: [
+    [
+      'catch-zht.html',
+      'https://dictionary.cambridge.org/zhs/%E8%AF%8D%E5%85%B8/%E8%8B%B1%E8%AF%AD-%E6%B1%89%E8%AF%AD-%E7%B9%81%E4%BD%93/catch'
+    ],
+    [
+      'house-zhs.html',
+      'https://dictionary.cambridge.org/zhs/%E8%AF%8D%E5%85%B8/%E8%8B%B1%E8%AF%AD-%E6%B1%89%E8%AF%AD-%E7%AE%80%E4%BD%93/house'
+    ],
+    [
+      'love.html',
+      'https://dictionary.cambridge.org/zhs/%E8%AF%8D%E5%85%B8/%E8%8B%B1%E8%AF%AD-%E6%B1%89%E8%AF%AD-%E7%AE%80%E4%BD%93/love'
+    ]
+  ]
+}

test/specs/components/dictionaries/cnki/fixtures.js

@@ -0,0 +1,5 @@
+module.exports = {
+  files: [
+    ['love.html', 'http://dict.cnki.net/dict_result.aspx?searchword=love']
+  ]
+}