fix: use absolute path for file download

Author: PaulKalho

.github/workflows/ci.yaml

@@ -38,6 +38,28 @@ jobs:
       - name: Intall dependencies
         run: |
           pip install -r requirements.txt
+      
+      - name: Debug directory structure
+        run: |
+          pwd
+          ls -R .
+      
+      - name: Print loaded CBCs
+        run: |
+          python3 - <<'EOF'
+          from scystream.sdk.config import load_config, get_compute_block
+          from pathlib import Path
+          import pprint
+
+          block_from_code = get_compute_block()
+          block_from_yaml = load_config("cbc.yaml")
+
+          print("Code block:")
+          pprint.pprint(block_from_code)
+
+          print("YAML block:")
+          pprint.pprint(block_from_yaml)
+          EOF
 
       - name: Check cbcs
         run: |

cbc.yaml

@@ -1,10 +1,11 @@
 author: Paul Kalhorn 
 description: Language preprocessing for .txt or .bib files
-docker_image: ghcr.io/rwth-time/language-preprocessing/language-preprocessing
+docker_image: ghcr.io/rwth-time/language-preprocessing/language-preprocessing 
 entrypoints:
   preprocess_bib_file:
-    description: Entrypoint for preprocessing a .bib file
+    description: Entrypoint for preprocessing a .bib file 
     envs:
+      BIB_DOWNLOAD_PATH: /tmp/input.bib
       FILTER_STOPWORDS: true
       LANGUAGE: en
       NGRAM_MAX: 3
@@ -23,7 +24,7 @@ entrypoints:
           bib_file_S3_PORT: null
           bib_file_S3_SECRET_KEY: null
           bib_file_SELECTED_ATTRIBUTE: Abstract
-        description: The bib file, aswell as one attribute selected for preprocessing 
+        description: The bib file, aswell as one attribute selected for preprocessing
         type: file
     outputs:
       dtm_output:
@@ -36,7 +37,7 @@ entrypoints:
           dtm_output_S3_HOST: null
           dtm_output_S3_PORT: null
           dtm_output_S3_SECRET_KEY: null
-        description: Numpy representation of document-term matrix as .pkl file 
+        description: Numpy representation of document-term matrix as .pkl file
         type: file
       vocab_output:
         config:
@@ -57,6 +58,7 @@ entrypoints:
       LANGUAGE: en
       NGRAM_MAX: 3
       NGRAM_MIN: 2
+      TXT_DOWNLOAD_PATH: /tmp/input.txt
       UNIGRAM_NORMALIZER: porter
       USE_NGRAMS: true
     inputs:
@@ -70,7 +72,7 @@ entrypoints:
           txt_file_S3_HOST: null
           txt_file_S3_PORT: null
           txt_file_S3_SECRET_KEY: null
-        description: A .txt file 
+        description: A .txt file
         type: file
     outputs:
       dtm_output:

input.bib

@@ -14,7 +14,6 @@
 from preprocessing.core import Preprocessor
 from preprocessing.loader import TxtLoader, BibLoader
 
-
 logging.basicConfig(
     level=logging.INFO,
     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
@@ -54,6 +53,8 @@ class PreprocessTXT(EnvSettings):
     NGRAM_MIN: int = 2
     NGRAM_MAX: int = 3
 
+    TXT_DOWNLOAD_PATH: str = "/tmp/input.txt"
+
     txt_input: TXTFileInput
     dtm_output: DTMFileOutput
     vocab_output: VocabFileOutput
@@ -67,6 +68,8 @@ class PreprocessBIB(EnvSettings):
     NGRAM_MIN: int = 2
     NGRAM_MAX: int = 3
 
+    BIB_DOWNLOAD_PATH: str = "/tmp/input.bib"
+
     bib_input: BIBFileInput
     dtm_output: DTMFileOutput
     vocab_output: VocabFileOutput
@@ -113,20 +116,20 @@ def _preprocess_and_store(texts, settings):
 @entrypoint(PreprocessTXT)
 def preprocess_txt_file(settings):
     logger.info("Downloading TXT input from S3...")
-    S3Operations.download(settings.txt_input, "input.txt")
+    S3Operations.download(settings.txt_input, settings.TXT_DOWNLOAD_PATH)
 
-    texts = TxtLoader.load("./input.txt")
+    texts = TxtLoader.load(settings.TXT_DOWNLOAD_PATH)
 
     _preprocess_and_store(texts, settings)
 
 
 @entrypoint(PreprocessBIB)
 def preprocess_bib_file(settings):
     logger.info("Downloading BIB input from S3...")
-    S3Operations.download(settings.bib_input, "input.bib")
+    S3Operations.download(settings.bib_input, settings.BIB_DOWNLOAD_PATH)
 
     texts = BibLoader.load(
-        "./input.bib",
+        settings.BIB_DOWNLOAD_PATH,
         attribute=settings.bib_input.SELECTED_ATTRIBUTE,
     )
     _preprocess_and_store(texts, settings)