vfdev-5
diff --git a/‎Dockerfile‎
Lines changed: 10 additions & 0 deletions b/‎Dockerfile‎
Lines changed: 10 additions & 0 deletions
diff --git a/‎tests/test_allennlp.py‎
Lines changed: 15 additions & 0 deletions b/‎tests/test_allennlp.py‎
Lines changed: 15 additions & 0 deletions
@@ -437,6 +437,16 @@ RUN pip install bcolz && \
     pip install torchtext && \
     /tmp/clean-layer.sh
 
+# allennlp and dependencies
+# TODO: install deps when underlying dependency is fixed. https://github.com/Kaggle/docker-python/issues/548
+RUN pip install jsonnet overrides tensorboardX && \
+    pip install flask>=1.0.2 flask-cors>=3.0.7 gevent>=1.3.6 && \
+    pip install unidecode parsimonious>=0.8.0 sqlparse>=0.2.4 word2number>=1.1 && \
+    pip install pytorch-pretrained-bert>=0.6.0 jsonpickle && \
+    pip install requests>=2.18 editdistance conllu==0.11 && \
+    pip install --no-dependencies allennlp && \
+    /tmp/clean-layer.sh
+
     ###########
     #
     #      NEW CONTRIBUTORS:
 
@@ -0,0 +1,15 @@
+import unittest
+
+from allennlp.data.tokenizers import WordTokenizer
+
+
+class TestAllenNlp(unittest.TestCase):
+    # reference
+    # https://github.com/allenai/allennlp/blob/master/allennlp/tests/data/tokenizers/word_tokenizer_test.py
+    def test_passes_through_correctly(self):
+        tokenizer = WordTokenizer(start_tokens=['@@', '%%'], end_tokens=['^^'])
+        sentence = "this (sentence) has 'crazy' \"punctuation\"."
+        tokens = [t.text for t in tokenizer.tokenize(sentence)]
+        expected_tokens = ["@@", "%%", "this", "(", "sentence", ")", "has", "'", "crazy", "'", "\"",
+                           "punctuation", "\"", ".", "^^"]
+        assert tokens == expected_tokens