Update gitignore and compose file

.gitignore

@@ -3,6 +3,11 @@
 **/.pytest_cache
 **/.mypy_cache
 **/.DS_Store
+*/.DS_Store
+*/**/.DS_Store
+.DS_Store
+.venv
+.idea
 *.pyc
 *.pyo
 *.pyd
@@ -14,6 +19,7 @@
 
 archived/
 qdrant_data/
+raw_documents/
 
 data/*
 !data/parsed_documents/

.gitlab-ci.yml

 stages:
+  - build
+  - export
   - deploy
 
-deploy:
+variables:
+  IMAGE_TAG: latest
+  RETRIEVAL_IMAGE: transferbot/retrieval:$IMAGE_TAG
+  INGESTION_IMAGE: transferbot/ingestion:$IMAGE_TAG
+  SSH_USER: huyminh
+  SSH_HOST: 94.130.88.60
+  SSH_TARGET_DIR: ~/hft-transferbot
+
+before_script:
+  - echo "Using tag: $IMAGE_TAG"
+
+build:retrieval:
+  stage: build
+  script:
+    - docker build --platform linux/amd64 -f retrieval/Dockerfile -t $RETRIEVAL_IMAGE .
+  tags: [docker]
+
+build:ingestion:
+  stage: build
+  script:
+    - docker build --platform linux/amd64 -f ingestion/Dockerfile -t $INGESTION_IMAGE .
+  tags: [docker]
+
+export:images:
+  stage: export
+  script:
+    - docker save -o transferbot-retrieval.tar $RETRIEVAL_IMAGE
+    - docker save -o transferbot-ingestion.tar $INGESTION_IMAGE
+  artifacts:
+    paths:
+      - transferbot-retrieval.tar
+      - transferbot-ingestion.tar
+
+deploy:scp:
   stage: deploy
   only:
-    - main
-  before_script:
+    - master
+  script:
     - mkdir -p ~/.ssh
     - echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519
     - chmod 600 ~/.ssh/id_ed25519
-    - ssh-keyscan -H $SERVER_IP >> ~/.ssh/known_hosts
-
-  script:
-    - scp backend-chatbot.tar .env docker-compose.yml $SERVER_USER@$SERVER_IP:~/hft-transferbot/
-    - ssh $SERVER_USER@$SERVER_IP "
-      cd ~/hft-transferbot &&
-      docker load -i backend-chatbot.tar &&
-      docker compose down &&
-      docker compose up -d
-      "
+    - ssh-keyscan -H $SSH_HOST >> ~/.ssh/known_hosts
+    - scp transferbot-retrieval.tar transferbot-ingestion.tar $SSH_USER@$SSH_HOST:$SSH_TARGET_DIR
+  dependencies:
+    - export:images
+  tags: [docker]

docker-compose.yml

@@ -6,6 +6,7 @@ services:
       - "6333:6333"
     volumes:
       - qdrant_db:/qdrant/storage
+    restart: always
 
   ollama:
     image: ollama/ollama
@@ -24,7 +25,8 @@ services:
     volumes:
       - ./data:/app/data
       - ./raw_documents:/app/raw_documents
-      - ./logs:/app/logs
+      - ./logs:/app/retrieval/logs
+      - ./.env:/app/.env
     depends_on:
       - qdrant
       - ollama
@@ -42,11 +44,12 @@ services:
     volumes:
       - ./data:/app/data
       - ./raw_documents:/app/raw_documents
+      - .env:/app/.env
     env_file:
       - .env
     working_dir: /app
     command: >
-      sh -c 'python main.py ./raw_documents/*.pdf'
+      python main.py --path ./raw_documents
     restart: on-failure
 
   frontend:
@@ -59,4 +62,4 @@ services:
     restart: always
 volumes:
   qdrant_db:
-  ollama_data:
+  ollama_data:
\ No newline at end of file