Numerous fixes

Author: clovis

setup.py

@@ -10,5 +10,5 @@
     author="The ARTFL Project",
     author_email="clovisgladstone@gmail.com",
     packages=["text_preprocessing", "text_preprocessing.lang"],
-    install_requires=["unidecode", "PyStemmer", "spacy>=3.6", "orjson", "requests", "lz4"],
+    install_requires=["unidecode", "PyStemmer", "spacy>=3.6", "orjson", "requests", "lz4", "regex"],
 )

text_preprocessing/__init__.py

@@ -1,4 +1,4 @@
 from .modernizer import Modernizer
-from .preprocessor import PreProcessor, Tokens, Token
+from .preprocessor import PreProcessor, Tokens, PreprocessorToken as Token
 from .text_loader import text_loader
 from .spacy_helpers import load_language_model