setup for mbart test

free-variation · Apr 29, 2021 · a9823a2 · a9823a2
1 parent ee5281d
commit a9823a2
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 1 deletion.
diff --git a/Makefile b/Makefile
@@ -93,3 +93,8 @@ build-pdl-corpus:
 	gawk 'BEGIN {FS="\t"} !/urn:/ {print $$3}' corpora/pdl/all.txt > corpora/pdl.txt
 
 build-corpora: download-openITI build-openITI-corpus download-pdl build-pdl-corpus
+
+
+
+nb:
+	jupyter notebook --NotebookApp.token='' --no-browser
diff --git a/requirements.txt b/requirements.txt
@@ -1 +1,6 @@
 git+https://github.com/mittagessen/kraken@master#egg=kraken
+transformers
+sentencepiece
+datasets
+tensorboardx
+
diff --git a/scripts/clean_openiti.py b/scripts/clean_openiti.py
@@ -1,3 +1,4 @@
+import sys
 import re
 from functools import partial
 
@@ -37,7 +38,6 @@ def _clean(s):
             (_clean(line) for line in lines) 
             if _has_arabic(cleaned_line))
 
-import sys
 if __name__ == '__main__':
 
     for filename in sys.argv[1:]: