update ingestion

.gitlab-ci.yml

@@ -82,11 +82,10 @@ generate:
     REDDIT_CLIENT_ID: "${REDDIT_CLIENT_ID}"
     REDDIT_CLIENT_SECRET: "${REDDIT_CLIENT_SECRET}"
     SUBREDDITS: "${SUBREDDITS}"
-    OUTPUT_DIR: "generated"
+    OUTPUT_DIR: "/generated"  # <- added shared VM folder
   before_script:
     - pip install -r requirements.txt
-    - mkdir -p generated
-    - mkdir -p /generated   # persistent folder on VM
+    - mkdir -p /generated
   script:
     - echo "Running data generator..."
     - python app.py
@@ -96,9 +95,6 @@ generate:
     - cp -R generated/. /generated/ || echo "Copy failed!"
     - echo "--- FILES NOW IN VM ---"
     - ls -Ra /generated/
-  artifacts:
-    paths: []
-    expire_in: "1 minute"
 
 # =====================================================================
 #                              INGEST
@@ -121,12 +117,8 @@ ingest:
     - ls -Ra /generated/ || echo "No files found!"
   script:
     - echo "Running ingestion..."
-    - java -jar plugin.jar "/generated/"
+    - java -jar plugin.jar "/generated"
     - echo "Ingestion done."
-  artifacts:
-    when: always
-    paths:
-      - logs/
 
 # =====================================================================
 #                              BUILD

tools/user-config.json

@@ -11,7 +11,7 @@
     },
     "filesystem": {
         "sourcePaths": [
-            "generated"
+            "/generated"
         ]
     }   
 }
\ No newline at end of file