Commit 6e77a284 authored by Benavides Ontiveros's avatar Benavides Ontiveros
Browse files

fix indent

parent 5d8b6c1c
Pipeline #12249 failed with stages
in 53 seconds
...@@ -16,7 +16,9 @@ variables: ...@@ -16,7 +16,9 @@ variables:
# BASE JOB TEMPLATE # BASE JOB TEMPLATE
# ===================================================================== # =====================================================================
.build_template: .build_template:
tags: ["ubuntu", "vm"] tags:
- ubuntu
- vm
image: docker:24.0.9 image: docker:24.0.9
before_script: before_script:
- echo "Using host Docker daemon" - echo "Using host Docker daemon"
...@@ -28,12 +30,14 @@ variables: ...@@ -28,12 +30,14 @@ variables:
lint: lint:
stage: lint stage: lint
image: python:3.10 image: python:3.10
tags: ["ubuntu", "vm"] tags:
- ubuntu
- vm
before_script: before_script:
- pip install flake8 - pip install flake8
script: | script:
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
# ===================================================================== # =====================================================================
# TEST # TEST
...@@ -41,80 +45,79 @@ lint: ...@@ -41,80 +45,79 @@ lint:
test: test:
stage: test stage: test
image: python:3.10 image: python:3.10
tags: ["ubuntu", "vm"] tags:
- ubuntu
- vm
before_script: before_script:
- pip install pytest - pip install pytest
script: pytest -q || echo "No tests found" script:
- pytest -q || echo "No tests found"
# ===================================================================== # =====================================================================
# DIAGNOSTICS # DIAG
# ===================================================================== # =====================================================================
diag: diag:
stage: diag stage: diag
extends: .build_template extends: .build_template
script: | script:
ls -l /var/run - ls -l /var/run
stat /var/run/docker.sock || true - stat /var/run/docker.sock || true
docker version || true - docker version || true
docker info || true - docker info || true
# ===================================================================== # =====================================================================
# GENERATE FILES # GENERATE
# ===================================================================== # =====================================================================
generate: generate:
stage: generate stage: generate
image: python:3.10 image: python:3.10
tags: ["ubuntu", "vm"] tags:
- ubuntu
- vm
variables: variables:
REDDIT_CLIENT_ID: ${REDDIT_CLIENT_ID} REDDIT_CLIENT_ID: "${REDDIT_CLIENT_ID}"
REDDIT_CLIENT_SECRET: ${REDDIT_CLIENT_SECRET} REDDIT_CLIENT_SECRET: "${REDDIT_CLIENT_SECRET}"
SUBREDDITS: ${SUBREDDITS} SUBREDDITS: "${SUBREDDITS}"
before_script: before_script:
- pip install -r requirements.txt - pip install -r requirements.txt
- mkdir -p generated - mkdir -p generated
- mkdir -p /generated/$CI_PIPELINE_ID # <-- persistent VM storage - mkdir -p /generated/$CI_PIPELINE_ID
script: |
echo "Running data generator..."
python app.py
echo "Copying generated files to the VM..." script:
cp -R generated/* /generated/$CI_PIPELINE_ID/ - echo "Running data generator..."
- python app.py
- echo "Copying generated files to VM..."
- cp -R generated/* /generated/$CI_PIPELINE_ID/
- echo "Saved to /opt/generated/$CI_PIPELINE_ID"
echo "VM files saved under:"
echo "/opt/generated/$CI_PIPELINE_ID (VM path)"
# REMOVE ALL ARTIFACTS
artifacts: artifacts:
paths: [] paths: []
expire_in: "1 minute" expire_in: "1 minute"
# ===================================================================== # =====================================================================
# INGEST DATA # INGEST
# ===================================================================== # =====================================================================
ingest: ingest:
stage: ingest stage: ingest
image: eclipse-temurin:17-jdk image: eclipse-temurin:17-jdk
tags: ["java", "ubuntu", "vm"] tags:
needs: [] # NO ARTIFACTS - java
- ubuntu
- vm
needs: []
before_script: before_script:
- apt-get update && apt-get install -y gettext jq - apt-get update && apt-get install -y gettext jq
- cp tools/filesystem-plugin-2.1.0.jar plugin.jar - cp tools/filesystem-plugin-2.1.0.jar plugin.jar
- envsubst < tools/user-config.json > user-config.json - envsubst < tools/user-config.json > user-config.json
- echo "Generated files on VM:"
- ls -R /generated/$CI_PIPELINE_ID || echo "No files found"
# Show config (mask password) script:
- cat user-config.json | sed 's/"password":".*"/"password":"****"/' - echo "Running ingestion..."
- java -jar plugin.jar "/generated/$CI_PIPELINE_ID"
echo "Reading generated files from VM..." - echo "Ingestion done."
- ls -R /generated/$CI_PIPELINE_ID || echo "No files found!"
script: |
echo "Running ingestion..."
java -jar plugin.jar "/generated/$CI_PIPELINE_ID"
echo "Ingestion finished."
artifacts: artifacts:
when: always when: always
...@@ -127,14 +130,12 @@ ingest: ...@@ -127,14 +130,12 @@ ingest:
build: build:
stage: build stage: build
extends: .build_template extends: .build_template
script: | script:
echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin - echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
- docker build \
docker build \ --build-arg REDDIT_CLIENT_ID="$REDDIT_CLIENT_ID" \
--build-arg REDDIT_CLIENT_ID="$REDDIT_CLIENT_ID" \ --build-arg REDDIT_CLIENT_SECRET="$REDDIT_CLIENT_SECRET" \
--build-arg REDDIT_CLIENT_SECRET="$REDDIT_CLIENT_SECRET" \ -t $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA .
-t $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA .
after_script: after_script:
- echo "Docker image built successfully!" - echo "Docker image built successfully!"
...@@ -144,12 +145,11 @@ build: ...@@ -144,12 +145,11 @@ build:
deploy: deploy:
stage: deploy stage: deploy
extends: .build_template extends: .build_template
script: | script:
echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin - echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin
- docker push $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA
docker push $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA - |
if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then docker tag $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA $DOCKERHUB_REPO:latest
docker tag $DOCKERHUB_REPO:$CI_COMMIT_SHORT_SHA $DOCKERHUB_REPO:latest docker push $DOCKERHUB_REPO:latest
docker push $DOCKERHUB_REPO:latest fi
fi \ No newline at end of file
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment