Merge pull request #3 from akvo/feature/2-seed-rag-web-ui-with-unep-documents

Feature/2 seed rag web UI with unep documents

Author: dedenbangkit

.env.example

@@ -65,3 +65,13 @@ ACCESS_TOKEN_EXPIRE_MINUTES=10080
 
 # Timezone settings (optional)
 TZ=Asia/Shanghai
+
+# DATA PIPELINE
+RAG_USERNAME="rag_admin"
+RAG_PASSWORD="RAGadmin1"
+
+# CUSTOM DEV ENV
+NGINX_PORT=80
+BACKEND_PORT=8000
+DB_PORT=3306
+CHROMADB_IMAGE_VERSION=latest

backend/app/api/api_v1/api_keys.py

@@ -11,7 +11,7 @@
 router = APIRouter()
 logger = logging.getLogger(__name__)
 
-@router.get("/", response_model=List[schemas.APIKey])
+@router.get("", response_model=List[schemas.APIKey])
 def read_api_keys(
     db: Session = Depends(get_db),
     skip: int = 0,
@@ -26,7 +26,7 @@ def read_api_keys(
     )
     return api_keys
 
-@router.post("/", response_model=schemas.APIKey)
+@router.post("", response_model=schemas.APIKey)
 def create_api_key(
     *,
     db: Session = Depends(get_db),

backend/app/api/api_v1/chat.py

@@ -18,7 +18,7 @@
 
 router = APIRouter()
 
-@router.post("/", response_model=ChatResponse)
+@router.post("", response_model=ChatResponse)
 def create_chat(
     *,
     db: Session = Depends(get_db),
@@ -51,7 +51,7 @@ def create_chat(
     db.refresh(chat)
     return chat
 
-@router.get("/", response_model=List[ChatResponse])
+@router.get("", response_model=List[ChatResponse])
 def get_chats(
     db: Session = Depends(get_db),
     current_user: User = Depends(get_current_user),

dev.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+COMPOSE_HTTP_TIMEOUT=180 docker compose \
+  -f docker-compose.dev.yml \
+  "$@"

docker-compose.dev.yml

@@ -2,7 +2,7 @@ services:
   nginx-dev:
     image: nginx:alpine
     ports:
-      - "80:80"
+      - "${NGINX_PORT}:80"
     volumes:
       - ./nginx.dev.conf:/etc/nginx/nginx.conf:ro
     depends_on:
@@ -28,7 +28,7 @@ services:
       context: ./backend
       dockerfile: Dockerfile.dev
     ports:
-      - "8000:8000"
+      - "${BACKEND_PORT}:8000"
     env_file:
       - .env
     environment:
@@ -63,7 +63,7 @@ services:
       - WATCHPACK_POLLING=true
       - CHOKIDAR_USEPOLLING=true
       - NODE_ENV=development
-      - NEXT_PUBLIC_API_URL=http://localhost/api
+      - NEXT_PUBLIC_API_URL=http://localhost:${NGINX_PORT}/api
     ports:
       - "3000:3000"
     volumes:
@@ -89,14 +89,15 @@ services:
       - MYSQL_PASSWORD=ragwebui
       - TZ=Asia/Shanghai
     ports:
-      - "3306:3306"
+      - "${DB_PORT}:3306"
     volumes:
       - mysql_data:/var/lib/mysql
     networks:
       - app_network
 
   chromadb:
-    image: chromadb/chroma:latest
+    image: chromadb/chroma:${CHROMADB_IMAGE_VERSION}
+    platform: linux/amd64
     ports:
       - "8001:8000"
     volumes:
@@ -118,6 +119,22 @@ services:
     networks:
       - app_network
 
+  script:
+    build:
+      context: ./script
+      dockerfile: Dockerfile
+    depends_on:
+      - backend
+    environment:
+      - RAG_USERNAME=${RAG_USERNAME}
+      - RAG_PASSWORD=${RAG_PASSWORD}
+    volumes:
+      - ./script:/app
+    tty: true
+    command: ["tail", "-f", "/dev/null"]
+    networks:
+      - app_network
+
 volumes:
   mysql_data:
   chroma_data:

script/.gitignore

@@ -0,0 +1 @@
+./downloads

script/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
+
+WORKDIR /app
+
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+
+COPY . /app
+
+# Set up the command to keep the container running
+CMD ["tail", "-f", "/dev/null"]

script/README.md

@@ -0,0 +1,51 @@
+# 📚 Table of Contents
+
+- [📚 Table of Contents](#-table-of-contents)
+- [🤖 UNEP Knowledge Base import script](#-unep-knowledge-base-import-script)
+  - [🔐 Environment Variables](#-environment-variables)
+  - [🚀 Running the Script](#-running-the-script)
+  - [📁 Directory Structure](#-directory-structure)
+
+---
+
+# 🤖 UNEP Knowledge Base import script
+
+This script automates the process of collecting, saving, and uploading PDF documents from [GlobalPlasticsHub](https://globalplasticshub.org) into a RAG (Retrieval-Augmented Generation) system.
+
+This Python script supports three main operation modes:
+
+1. **CSV Only** – Save PDF URLs to a CSV file.
+2. **CSV + Download** – Save URLs and download the corresponding PDFs.
+3. **Full Process** – Save URLs, download PDFs, and upload/process them in RAG.
+
+## 🔐 Environment Variables
+
+Before running the script, set RAG credentials in your shell or environment:
+
+``` bash
+export RAG_USERNAME="rag_admin"
+export RAG_PASSWORD="RAGadmin1"
+```
+
+## 🚀 Running the Script
+
+To execute the script:
+
+```bash
+./dev.sh exec script python -m kb_init_unep
+```
+
+You will be prompted to:
+- Choose the operation mode:
+  1: Save PDF URLs to CSV only.
+  2: Save to CSV and download PDFs.
+  3: Full process (CSV + download + upload to RAG).
+
+- Enter the number of documents to import.
+- Provide a description for the RAG knowledge base.
+
+## 📁 Directory Structure
+```bash
+./downloads/unep/unep_files.csv – Stores PDF URLs and offsets.
+./downloads/unep/ – Folder where downloaded PDF files are saved.
+```