modify ingest pipelines

.gitlab-ci-ingest.yml

@@ -4,9 +4,7 @@
 
 workflow:
   rules:
-    - if: '$CI_PIPELINE_SOURCE == "web"'
-      when: always
-    - when: never
+    - when: always     # REQUIRED so GitLab Free loads this pipeline
 
 stages:
   - ingest
@@ -21,7 +19,7 @@ ingest:
     - reddit
 
   rules:
-    - when: manual
+    - when: manual     # must click the Play button
 
   before_script:
     - apt-get update && apt-get install -y gettext jq

.gitlab-ci.yml

@@ -4,6 +4,12 @@ workflow:
       when: always
     - when: never
 
+# REQUIRED so GitLab Free recognizes the second pipeline file
+include:
+  - local: '.gitlab-ci-ingest.yml'
+    rules:
+      - when: never      # Do not run automatically
+
 stages:
   - lint
   - test
@@ -19,7 +25,7 @@ variables:
   DOCKERHUB_REPO: "$DOCKER_USERNAME/reddit_data_fetcher"
 
 # =====================================================================
-#                     BASE JOB TEMPLATE
+# BASE JOB TEMPLATE
 # =====================================================================
 .build_template:
   tags:
@@ -32,7 +38,7 @@ variables:
     - docker info || true
 
 # =====================================================================
-#                             LINT
+# LINT
 # =====================================================================
 lint:
   stage: lint
@@ -48,7 +54,7 @@ lint:
     - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
 # =====================================================================
-#                             TEST
+# TEST
 # =====================================================================
 test:
   stage: test
@@ -63,7 +69,7 @@ test:
     - pytest -q || echo "No tests found"
 
 # =====================================================================
-#                             DIAG
+# DIAG
 # =====================================================================
 diag:
   stage: diag
@@ -75,7 +81,7 @@ diag:
     - docker info || true
 
 # =====================================================================
-#                          GENERATE
+# GENERATE
 # =====================================================================
 generate:
   stage: generate
@@ -95,15 +101,12 @@ generate:
   script:
     - echo "Running data generator..."
     - python app.py
-    - echo "--- GENERATED (local container folder) ---"
     - ls -Ra generated/ || echo "No files generated"
-    - echo "Copying to VM mapped folder..."
     - cp -R generated/. /reddit_output/ || echo "Copy failed!"
-    - echo "--- VM FOLDER CONTENTS ---"
     - ls -Ra /reddit_output/
 
 # =====================================================================
-#                              INGEST
+# INGEST
 # =====================================================================
 ingest:
   stage: ingest
@@ -115,22 +118,16 @@ ingest:
     - reddit
   needs:
     - job: generate
-    
+
   before_script:
     - apt-get update && apt-get install -y gettext jq
     - cp tools/filesystem-plugin-2.1.0.jar plugin.jar
     - envsubst < tools/user-config.json > user-config.json
-
-    - echo "--- Listing files in /reddit_output ---"
     - ls -Ra /reddit_output/ || echo "No files found!"
-
-    - echo "--- Listing files in /reddit_output/.meta ---"
     - ls -Ra /reddit_output/.meta || echo ".meta folder not found!"
 
   script:
-    - echo "Running ingestion..."
     - java -jar plugin.jar "/reddit_output"
-    - echo "Ingestion done."
 
 # =====================================================================
 #                              BUILD