test ingestion

.gitlab-ci.yml

@@ -131,6 +131,46 @@ build:
     - if: '$CI_COMMIT_TAG'
       when: never
 
+# =====================================================================
+#                              INGEST DATA
+# =====================================================================
+
+ingest:
+  stage: ingest
+  image: openjdk:17-jdk
+  tags: ["ubuntu", "vm"]
+  needs:
+    - job: generate
+      artifacts: true
+
+  before_script:
+    # Copy plugin JAR from repo
+    - cp tools/filesystem-plugin-2.1.0.jar plugin.jar
+
+    # Build final config file by replacing environment variables
+    - envsubst < tools/bigdata-config-template.json > bigdata-config.json
+
+    # Show config (without password)
+    - cat user-config.json | sed 's/"password":".*"/"password":"****"/'
+
+    # Show generated files
+    - echo "Generated files:"
+    - ls -R generated/
+
+  script: |
+    echo "Running BigData4Biz ingestion..."
+    java -jar plugin.jar bigdata-config.json
+    echo "Ingestion finished."
+
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+
+  artifacts:
+    when: always
+    paths:
+      - logs/
+
+
 # =====================================================================
 #                              DEPLOY
 # =====================================================================

tools/user-config.json

+{
+    "pluginConfig": {
+        "datasource": "metadata-test-002",
+        "pluginId": "dab844fe-bdb5-11f0-8de9-0242ac120002"
+    },
+    "platformConfig": {
+        "url": "https://portal.bigdata4biz.com",
+        "username": "$BIGDATA_USERNAME",
+        "password": "$BIGDATA_PASSWORD",
+        "tenant": "$BIGDATA_TENANT"
+    },
+    "filesystem": {
+        "sourcePaths": [
+            "generated"
+        ]
+    }   
+}
\ No newline at end of file