Commit 8700c0c9 authored by Benavides Ontiveros's avatar Benavides Ontiveros
Browse files

modify ingest pipeline

parent b9cc5c4b
# ===========================================
# INGEST PIPELINE
# ===========================================
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "web"'
when: always # allow manual run from the UI
- when: never # block all automatic runs
when: always
- when: never
stages:
- ingest
......@@ -17,20 +21,12 @@ ingest:
- reddit
rules:
- when: manual # job requires clicking the play button
- when: manual
before_script:
- apt-get update && apt-get install -y gettext jq
- cp tools/filesystem-plugin-2.1.0.jar plugin.jar
- envsubst < tools/user-config.json > user-config.json
- echo "--- Listing files in /reddit_output ---"
- ls -Ra /reddit_output/ || echo "No files found!"
- echo "--- Listing files in /reddit_output/.meta ---"
- ls -Ra /reddit_output/.meta || echo ".meta folder not found!"
script:
- echo "Running ingestion..."
- java -jar plugin.jar "/reddit_output"
- echo "Ingestion done."
\ No newline at end of file
- java -jar plugin.jar "/reddit_output"
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment