remove gitlab artifacts

.gitlab-ci.yml

@@ -34,11 +34,6 @@ lint:
   script: |
     flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
     flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-    - if: '$CI_COMMIT_BRANCH'
-    - if: '$CI_COMMIT_TAG'
-      when: never
 
 # =====================================================================
 #                             TEST
@@ -50,11 +45,6 @@ test:
   before_script:
     - pip install pytest
   script: pytest -q || echo "No tests found"
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-    - if: '$CI_COMMIT_BRANCH'
-    - if: '$CI_COMMIT_TAG'
-      when: never
 
 # =====================================================================
 #                          DIAGNOSTICS
@@ -67,11 +57,6 @@ diag:
     stat /var/run/docker.sock || true
     docker version || true
     docker info || true
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-    - if: '$CI_COMMIT_BRANCH'
-    - if: '$CI_COMMIT_TAG'
-      when: never
 
 # =====================================================================
 #                          GENERATE FILES
@@ -85,69 +70,56 @@ generate:
     REDDIT_CLIENT_ID: ${REDDIT_CLIENT_ID}
     REDDIT_CLIENT_SECRET: ${REDDIT_CLIENT_SECRET}
     SUBREDDITS: ${SUBREDDITS}
-    OUTPUT_DIR: "generated"    # optional
 
   before_script:
     - pip install -r requirements.txt
     - mkdir -p generated
+    - mkdir -p /generated/$CI_PIPELINE_ID   # <-- persistent VM storage
 
   script: |
-    echo "Environment variables received:"
-    echo "REDDIT_CLIENT_ID: ${REDDIT_CLIENT_ID:0:4}****"
-    echo "REDDIT_CLIENT_SECRET: ${REDDIT_CLIENT_SECRET:0:4}****"
-    echo "SUBREDDITS: $SUBREDDITS"
-
+    echo "Running data generator..."
     python app.py
 
-  artifacts:
-    paths:
-      - generated/
+    echo "Copying generated files to the VM..."
+    cp -R generated/* /generated/$CI_PIPELINE_ID/
 
-  rules:
-    - if: '$CI_COMMIT_BRANCH'
+    echo "VM files saved under:"
+    echo "/opt/generated/$CI_PIPELINE_ID (VM path)"
+
+  # REMOVE ALL ARTIFACTS
+  artifacts:
+    paths: []
+    expire_in: "1 minute"
 
 # =====================================================================
 #                              INGEST DATA
 # =====================================================================
-
 ingest:
   stage: ingest
   image: eclipse-temurin:17-jdk
-  tags: ["java","ubuntu", "vm"]
-  needs:
-    - job: generate
-      artifacts: true
+  tags: ["java", "ubuntu", "vm"]
+  needs: []   # NO ARTIFACTS
 
   before_script:
-    # Install envsubst (gettext) and any other necessary tools
     - apt-get update && apt-get install -y gettext jq
-    # Copy plugin JAR from repo
     - cp tools/filesystem-plugin-2.1.0.jar plugin.jar
-    # Build final config file by replacing environment variables
     - envsubst < tools/user-config.json > user-config.json
+
     # Show config (mask password)
     - cat user-config.json | sed 's/"password":".*"/"password":"****"/'
-    # Show generated files
-    - echo "Generated files:"
-    - ls -R generated/
+
+    echo "Reading generated files from VM..."
+    - ls -R /generated/$CI_PIPELINE_ID || echo "No files found!"
 
   script: |
-    echo "Running BigData4Biz ingestion..."
-    java -jar plugin.jar
+    echo "Running ingestion..."
+    java -jar plugin.jar "/generated/$CI_PIPELINE_ID"
     echo "Ingestion finished."
 
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      when: never
-    - if: '$CI_COMMIT_BRANCH'
-    - if: '$CI_COMMIT_TAG'
-      when: never
-
   artifacts:
     when: always
     paths:
       - logs/
-      - generated/
 
 # =====================================================================
 #                              BUILD
@@ -155,11 +127,9 @@ ingest:
 build:
   stage: build
   extends: .build_template
-
   script: |
     echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
 
-    echo "Building Docker image for commit: ${CI_COMMIT_SHORT_SHA}"
     docker build \
       --build-arg REDDIT_CLIENT_ID="$REDDIT_CLIENT_ID" \
       --build-arg REDDIT_CLIENT_SECRET="$REDDIT_CLIENT_SECRET" \
@@ -168,13 +138,6 @@ build:
   after_script:
     - echo "Docker image built successfully!"
 
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      when: never
-    - if: '$CI_COMMIT_BRANCH'
-    - if: '$CI_COMMIT_TAG'
-      when: never
-
 # =====================================================================
 #                              DEPLOY
 # =====================================================================
@@ -184,14 +147,9 @@ deploy:
   script: |
     echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
 
-    docker info | grep Username || (echo "Docker login failed!" && exit 1)
-
     docker push $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA
 
     if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
       docker tag $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA $DOCKERHUB_REPO:latest
       docker push $DOCKERHUB_REPO:latest
-    fi
-
+    fi
\ No newline at end of file
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
\ No newline at end of file