Commit 6e77a284 authored by Benavides Ontiveros's avatar Benavides Ontiveros
Browse files

fix indent

parent 5d8b6c1c
Pipeline #12249 failed with stages
in 53 seconds
......@@ -16,7 +16,9 @@ variables:
# BASE JOB TEMPLATE
# =====================================================================
.build_template:
tags: ["ubuntu", "vm"]
tags:
- ubuntu
- vm
image: docker:24.0.9
before_script:
- echo "Using host Docker daemon"
......@@ -28,12 +30,14 @@ variables:
lint:
stage: lint
image: python:3.10
tags: ["ubuntu", "vm"]
tags:
- ubuntu
- vm
before_script:
- pip install flake8
script: |
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
script:
- flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
# =====================================================================
# TEST
......@@ -41,80 +45,79 @@ lint:
test:
stage: test
image: python:3.10
tags: ["ubuntu", "vm"]
tags:
- ubuntu
- vm
before_script:
- pip install pytest
script: pytest -q || echo "No tests found"
script:
- pytest -q || echo "No tests found"
# =====================================================================
# DIAGNOSTICS
# DIAG
# =====================================================================
diag:
stage: diag
extends: .build_template
script: |
ls -l /var/run
stat /var/run/docker.sock || true
docker version || true
docker info || true
script:
- ls -l /var/run
- stat /var/run/docker.sock || true
- docker version || true
- docker info || true
# =====================================================================
# GENERATE FILES
# GENERATE
# =====================================================================
generate:
stage: generate
image: python:3.10
tags: ["ubuntu", "vm"]
tags:
- ubuntu
- vm
variables:
REDDIT_CLIENT_ID: ${REDDIT_CLIENT_ID}
REDDIT_CLIENT_SECRET: ${REDDIT_CLIENT_SECRET}
SUBREDDITS: ${SUBREDDITS}
REDDIT_CLIENT_ID: "${REDDIT_CLIENT_ID}"
REDDIT_CLIENT_SECRET: "${REDDIT_CLIENT_SECRET}"
SUBREDDITS: "${SUBREDDITS}"
before_script:
- pip install -r requirements.txt
- mkdir -p generated
- mkdir -p /generated/$CI_PIPELINE_ID # <-- persistent VM storage
script: |
echo "Running data generator..."
python app.py
- mkdir -p /generated/$CI_PIPELINE_ID
echo "Copying generated files to the VM..."
cp -R generated/* /generated/$CI_PIPELINE_ID/
script:
- echo "Running data generator..."
- python app.py
- echo "Copying generated files to VM..."
- cp -R generated/* /generated/$CI_PIPELINE_ID/
- echo "Saved to /opt/generated/$CI_PIPELINE_ID"
echo "VM files saved under:"
echo "/opt/generated/$CI_PIPELINE_ID (VM path)"
# REMOVE ALL ARTIFACTS
artifacts:
paths: []
expire_in: "1 minute"
# =====================================================================
# INGEST DATA
# INGEST
# =====================================================================
ingest:
stage: ingest
image: eclipse-temurin:17-jdk
tags: ["java", "ubuntu", "vm"]
needs: [] # NO ARTIFACTS
tags:
- java
- ubuntu
- vm
needs: []
before_script:
- apt-get update && apt-get install -y gettext jq
- cp tools/filesystem-plugin-2.1.0.jar plugin.jar
- envsubst < tools/user-config.json > user-config.json
- echo "Generated files on VM:"
- ls -R /generated/$CI_PIPELINE_ID || echo "No files found"
# Show config (mask password)
- cat user-config.json | sed 's/"password":".*"/"password":"****"/'
echo "Reading generated files from VM..."
- ls -R /generated/$CI_PIPELINE_ID || echo "No files found!"
script: |
echo "Running ingestion..."
java -jar plugin.jar "/generated/$CI_PIPELINE_ID"
echo "Ingestion finished."
script:
- echo "Running ingestion..."
- java -jar plugin.jar "/generated/$CI_PIPELINE_ID"
- echo "Ingestion done."
artifacts:
when: always
......@@ -127,14 +130,12 @@ ingest:
build:
stage: build
extends: .build_template
script: |
echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
docker build \
--build-arg REDDIT_CLIENT_ID="$REDDIT_CLIENT_ID" \
--build-arg REDDIT_CLIENT_SECRET="$REDDIT_CLIENT_SECRET" \
-t $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA .
script:
- echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
- docker build \
--build-arg REDDIT_CLIENT_ID="$REDDIT_CLIENT_ID" \
--build-arg REDDIT_CLIENT_SECRET="$REDDIT_CLIENT_SECRET" \
-t $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA .
after_script:
- echo "Docker image built successfully!"
......@@ -144,12 +145,11 @@ build:
deploy:
stage: deploy
extends: .build_template
script: |
echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
docker push $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA
if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
docker tag $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA $DOCKERHUB_REPO:latest
docker push $DOCKERHUB_REPO:latest
fi
\ No newline at end of file
script:
- echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
- docker push $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA
- |
if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
docker tag $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA $DOCKERHUB_REPO:latest
docker push $DOCKERHUB_REPO:latest
fi
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment