forked from IBM/multidoc2dial
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_converter.sh
34 lines (25 loc) · 1.02 KB
/
run_converter.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
#!/bin/sh
domain=$1
seg=$2
config=dpr-$domain-$seg
dpr=dpr_mdd-$domain-$seg
src=YOUR_DPR_CHECKPOINT
mkdir $CHECKPOINTS/$config
python convert_dpr_original_checkpoint_to_pytorch.py \
--type question_encoder \
--src $src \
--dest $CHECKPOINTS/dpr-$domain-$seg/question_encoder
python convert_dpr_original_checkpoint_to_pytorch.py \
--type ctx_encoder \
--src $src \
--dest $CHECKPOINTS/dpr-$domain-$seg/ctx_encoder
# generate rag model
cp ../data/tokenizer_config.json $CHECKPOINTS/$config/question_encoder/
cp ../data/vocab.txt $CHECKPOINTS/$config/question_encoder/
cp ../data/tokenizer_config.json $CHECKPOINTS/$config/ctx_encoder/
cp ../data/vocab.txt $CHECKPOINTS/$config/ctx_encoder/
# config "model_path" for question encoder to your local path to DPR encoder;
# or use our uploaded model, such as "sivasankalpp/dpr-multidoc2dial-token-question-encoder" or "sivasankalpp/dpr-multidoc2dial-structure-question-encoder"
python model_converter.py \
--model_path $CHECKPOINTS/$config/question_encoder \
--out_path $CHECKPOINTS/rag-$config