chore: add logging

Author: PaulKalho

main.py

@@ -1,5 +1,6 @@
 import pickle
 import tempfile
+import logging
 
 from scystream.sdk.core import entrypoint
 from scystream.sdk.env.settings import (
@@ -14,6 +15,13 @@
 from preprocessing.loader import TxtLoader, BibLoader
 
 
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
 class DTMFileOutput(FileSettings, OutputSettings):
     __identifier__ = "dtm_output"
 
@@ -66,6 +74,8 @@ class PreprocessBIB(EnvSettings):
 
 def _preprocess_and_store(texts, settings):
     """Shared preprocessing logic for TXT and BIB."""
+    logger.info(f"Starting preprocessing with {len(texts)} documents")
+
     pre = Preprocessor(
         language=settings.LANGUAGE,
         filter_stopwords=settings.FILTER_STOPWORDS,
@@ -74,10 +84,12 @@ def _preprocess_and_store(texts, settings):
         ngram_min=settings.NGRAM_MIN,
         ngram_max=settings.NGRAM_MAX,
     )
-    pre.texts = texts
 
+    pre.texts = texts
     pre.analyze_texts()
+
     pre.generate_bag_of_words()
+
     dtm, vocab = pre.generate_document_term_matrix()
 
     with tempfile.NamedTemporaryFile(suffix="_dtm.pkl") as tmp_dtm, \
@@ -89,20 +101,30 @@ def _preprocess_and_store(texts, settings):
         pickle.dump(vocab, tmp_vocab)
         tmp_vocab.flush()
 
+        logger.info("Uploading DTM to S3...")
         S3Operations.upload(settings.dtm_output, tmp_dtm.name)
+
+        logger.info("Uploading vocabulary to S3...")
         S3Operations.upload(settings.vocab_output, tmp_vocab.name)
 
+    logger.info("Preprocessing completed successfully.")
+
 
 @entrypoint(PreprocessTXT)
 def preprocess_txt_file(settings):
+    logger.info("Downloading TXT input from S3...")
     S3Operations.download(settings.txt_input, "input.txt")
+
     texts = TxtLoader.load("./input.txt")
+
     _preprocess_and_store(texts, settings)
 
 
 @entrypoint(PreprocessBIB)
 def preprocess_bib_file(settings):
+    logger.info("Downloading BIB input from S3...")
     S3Operations.download(settings.bib_input, "input.bib")
+
     texts = BibLoader.load(
         "./input.bib",
         attribute=settings.bib_input.SELECTED_ATTRIBUTE,

preprocessing/core.py

@@ -1,3 +1,4 @@
+import logging
 import spacy
 import numpy as np
 
@@ -9,6 +10,7 @@
     "en": "en_core_web_sm",
     "de": "de_core_news_sm"
 }
+logger = logging.getLogger(__name__)
 
 
 class Preprocessor:
@@ -21,6 +23,12 @@ def __init__(
         ngram_min: int = 2,
         ngram_max: int = 3,
     ):
+        logger.info(
+            "Init Preprocessor (lang=%s, filter_stopwords=%s, ngrams=%s)",
+            language,
+            filter_stopwords,
+            use_ngrams,
+        )
         self.language = language
         self.filter_stopwords = filter_stopwords
         self.unigram_normalizer = unigram_normalizer
@@ -58,6 +66,7 @@ def filter_tokens(
         ]
 
     def analyze_texts(self):
+        logger.info(f"Analyzing {len(self.texts)} texts...")
         porter = PorterStemmer()
         for text in self.texts:
             doc = self.nlp(text)
@@ -67,8 +76,8 @@ def analyze_texts(self):
 
             for sentence in doc.sents:
                 filtered_tokens = self.filter_tokens(
-                        list(sentence),
-                        self.filter_stopwords
+                    list(sentence),
+                    self.filter_stopwords
                 )
                 normalized_tokens = [
                     self.normalize_token(t, porter) for t in filtered_tokens
@@ -93,6 +102,10 @@ def analyze_texts(self):
             if ngram_list:
                 self.ngram_frequency.update(ngram_list)
                 self.ngram_document_frequency.update(set(ngram_list))
+        logger.info(
+            f"Finished analyzing texts: {self.token_frequency} unigrams, {
+                self.ngram_frequency} n-grams",
+        )
 
     def normalize_token(
         self,
@@ -110,6 +123,7 @@ def normalize_token(
         return word
 
     def generate_bag_of_words(self):
+        logger.info("Generating bag-of-words...")
         porter = PorterStemmer()
         self.bag_of_words = []
 
@@ -177,7 +191,7 @@ def generate_document_term_matrix(self) -> (np.ndarray, dict):
             dtm (np.ndarray): shape = (num_docs, num_terms)
             vocab (dict): mapping term -> column index
         """
-
+        logger.info("Building document-term-matrix...")
         all_terms = set()
         for doc in self.bag_of_words:
             for t in doc:
@@ -194,4 +208,5 @@ def generate_document_term_matrix(self) -> (np.ndarray, dict):
                 term_idx = vocab[token["term"]]
                 dtm[doc_idx, term_idx] += 1
 
+        logger.info(f"Matrix shape: {dtm.shape} | Vocab size: {len(vocab)}")
         return dtm, vocab

preprocessing/loader.py

@@ -1,6 +1,9 @@
+import logging
 import re
 import bibtexparser
 
+logger = logging.getLogger(__name__)
+
 
 def normalize_text(text: str) -> str:
     if not text:
@@ -24,6 +27,7 @@ def normalize_text(text: str) -> str:
 class TxtLoader:
     @staticmethod
     def load(file_path: str) -> list[str]:
+        logger.info("Loading TXT file...")
         with open(file_path, "r", encoding="utf-8") as f:
             lines = f.readlines()
         return [normalize_text(line) for line in lines]
@@ -32,6 +36,7 @@ def load(file_path: str) -> list[str]:
 class BibLoader:
     @staticmethod
     def load(file_path: str, attribute: str) -> list[str]:
+        logger.info(f"Loading BIB file (attribute={attribute})...")
         with open(file_path, "r", encoding="utf-8") as f:
             bib_database = bibtexparser.load(f)