fix to output for non sent objects

Author: clovis

text_preprocessing/preprocessor.py

@@ -17,7 +17,7 @@
 from spacy.tokens import Doc, Token
 
 from .modernizer import Modernizer
-from .spacy_helpers import load_language_model, Tokens, PreprocessorToken
+from .spacy_helpers import PreprocessorToken, Tokens, load_language_model
 
 Doc.set_extension("metadata", default={})
 Doc.set_extension("char_num", default=0)
@@ -176,8 +176,8 @@ def process_texts(
                 if self.ngram_config is not None:
                     tokens = generate_ngrams(**self.ngram_config, tokens=tokens)
                 if self.post_func is not None:
-                    processed_doc = self.post_func(tokens)
-                    yield processed_doc
+                    tokens = self.post_func(tokens)
+                yield tokens
             elif isinstance(tokens, Doc):
                 tokens = Tokens(tokens, keep_all=keep_all)
                 if self.ngram_config is not None: