modify volumes

.gitlab-ci.yml

@@ -19,6 +19,7 @@ variables:
   tags:
     - ubuntu
     - vm
+    - reddit          # Added Reddit VM tag
   image: docker:24.0.9
   before_script:
     - echo "Using host Docker daemon"
@@ -33,6 +34,7 @@ lint:
   tags:
     - ubuntu
     - vm
+    - reddit
   before_script:
     - pip install flake8
   script:
@@ -48,6 +50,7 @@ test:
   tags:
     - ubuntu
     - vm
+    - reddit
   before_script:
     - pip install pytest
   script:
@@ -74,23 +77,24 @@ generate:
   tags:
     - ubuntu
     - vm
+    - reddit
   variables:
     REDDIT_CLIENT_ID: "${REDDIT_CLIENT_ID}"
     REDDIT_CLIENT_SECRET: "${REDDIT_CLIENT_SECRET}"
     SUBREDDITS: "${SUBREDDITS}"
-
+    OUTPUT_DIR: "generated"
   before_script:
     - pip install -r requirements.txt
     - mkdir -p generated
     - mkdir -p /generated/$CI_PIPELINE_ID
-
   script:
     - echo "Running data generator..."
     - python app.py
+    - echo "--- GENERATED FILES INSIDE JOB ---"
+    - ls -R generated/ || echo "No files generated"
     - echo "Copying generated files to VM..."
-    - cp -R generated/* /generated/$CI_PIPELINE_ID/
-    - echo "Saved to /opt/generated/$CI_PIPELINE_ID"
-
+    - cp -R generated/* /generated/$CI_PIPELINE_ID/ || echo "Copy failed!"
+    - echo "Files saved to /opt/generated/$CI_PIPELINE_ID"
   artifacts:
     paths: []
     expire_in: "1 minute"
@@ -105,20 +109,19 @@ ingest:
     - java
     - ubuntu
     - vm
-  needs: []
-
+    - reddit
+  needs:
+    - job: generate
   before_script:
     - apt-get update && apt-get install -y gettext jq
     - cp tools/filesystem-plugin-2.1.0.jar plugin.jar
     - envsubst < tools/user-config.json > user-config.json
     - echo "Generated files on VM:"
-    - ls -R /generated/$CI_PIPELINE_ID || echo "No files found"
-
+    - ls -R /generated/$CI_PIPELINE_ID || echo "No files found!"
   script:
     - echo "Running ingestion..."
     - java -jar plugin.jar "/generated/$CI_PIPELINE_ID"
     - echo "Ingestion done."
-
   artifacts:
     when: always
     paths:
@@ -130,6 +133,10 @@ ingest:
 build:
   stage: build
   extends: .build_template
+  tags:
+    - ubuntu
+    - vm
+    - reddit
   script:
     - echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
     - docker build \
@@ -145,6 +152,10 @@ build:
 deploy:
   stage: deploy
   extends: .build_template
+  tags:
+    - ubuntu
+    - vm
+    - reddit
   script:
     - echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
     - docker push $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA