Commit 8700c0c9 authored by Benavides Ontiveros's avatar Benavides Ontiveros
Browse files

modify ingest pipeline

parent b9cc5c4b
# ===========================================
# INGEST PIPELINE
# ===========================================
workflow: workflow:
rules: rules:
- if: '$CI_PIPELINE_SOURCE == "web"' - if: '$CI_PIPELINE_SOURCE == "web"'
when: always # allow manual run from the UI when: always
- when: never # block all automatic runs - when: never
stages: stages:
- ingest - ingest
...@@ -17,20 +21,12 @@ ingest: ...@@ -17,20 +21,12 @@ ingest:
- reddit - reddit
rules: rules:
- when: manual # job requires clicking the play button - when: manual
before_script: before_script:
- apt-get update && apt-get install -y gettext jq - apt-get update && apt-get install -y gettext jq
- cp tools/filesystem-plugin-2.1.0.jar plugin.jar - cp tools/filesystem-plugin-2.1.0.jar plugin.jar
- envsubst < tools/user-config.json > user-config.json - envsubst < tools/user-config.json > user-config.json
- echo "--- Listing files in /reddit_output ---"
- ls -Ra /reddit_output/ || echo "No files found!"
- echo "--- Listing files in /reddit_output/.meta ---"
- ls -Ra /reddit_output/.meta || echo ".meta folder not found!"
script: script:
- echo "Running ingestion..." - java -jar plugin.jar "/reddit_output"
- java -jar plugin.jar "/reddit_output" \ No newline at end of file
- echo "Ingestion done."
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment