add metadata folder

.gitlab-ci.yml

@@ -86,15 +86,16 @@ generate:
   before_script:
     - pip install -r requirements.txt
     - mkdir -p generated
-    - mkdir -p /generated/$CI_PIPELINE_ID
+    - mkdir -p /generated/$CI_PIPELINE_ID   # persistent folder on VM
   script:
     - echo "Running data generator..."
     - python app.py
     - echo "--- GENERATED FILES INSIDE JOB ---"
-    - ls -R generated/ || echo "No files generated"
-    - echo "Copying generated files to VM..."
-    - cp -R generated/* /generated/$CI_PIPELINE_ID/ || echo "Copy failed!"
-    - echo "Files saved to /opt/generated/$CI_PIPELINE_ID"
+    - ls -Ra generated/ || echo "No files generated"
+    - echo "Copying all files (including .meta) to VM..."
+    - cp -R generated/. /generated/$CI_PIPELINE_ID/ || echo "Copy failed!"
+    - echo "--- FILES NOW IN VM ---"
+    - ls -Ra /generated/$CI_PIPELINE_ID
   artifacts:
     paths: []
     expire_in: "1 minute"
@@ -116,8 +117,8 @@ ingest:
     - apt-get update && apt-get install -y gettext jq
     - cp tools/filesystem-plugin-2.1.0.jar plugin.jar
     - envsubst < tools/user-config.json > user-config.json
-    - echo "Generated files on VM:"
-    - ls -R /generated/$CI_PIPELINE_ID || echo "No files found!"
+    - echo "--- GENERATED FILES ON VM ---"
+    - ls -Ra /generated/$CI_PIPELINE_ID || echo "No files found!"
   script:
     - echo "Running ingestion..."
     - java -jar plugin.jar "/generated/$CI_PIPELINE_ID"