diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index f6906f2..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -# 默认忽略的文件 -/shelf/ -/workspace.xml -# 基于编辑器的 HTTP 客户端请求 -/httpRequests/ -# 已忽略包含查询文件的默认文件夹 -/queries/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..eec9ad2 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/iStore_Docker_Compose.iml b/.idea/market-api.iml similarity index 54% rename from .idea/iStore_Docker_Compose.iml rename to .idea/market-api.iml index d0876a7..db670ba 100644 --- a/.idea/iStore_Docker_Compose.iml +++ b/.idea/market-api.iml @@ -1,8 +1,7 @@ - + - - + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml index 8eb3a06..efef7b6 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -3,6 +3,7 @@ + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..fb25c9f --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + { + "associatedIndex": 2, + "fromUser": false +} + + + + { + "keyToString": { + "ASKED_SHARE_PROJECT_CONFIGURATION_FILES": "true", + "RunOnceActivity.ShowReadmeOnStart": "true", + "RunOnceActivity.git.unshallow": "true", + "RunOnceActivity.typescript.service.memoryLimit.init": "true", + "SHARE_PROJECT_CONFIGURATION_FILES": "true", + "codeWithMe.voiceChat.enabledByDefault": "false", + "git-widget-placeholder": "main", + "node.js.detected.package.jshint": "true", + "node.js.selected.package.jshint": "", + "nodejs_package_manager_path": "npm" + } +} + + + + + + + + + + 1785115064124 + + + + + + \ No newline at end of file diff --git a/480T/RAGFlow/docker-compose.yaml b/480T/RAGFlow/docker-compose.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/480T/ragflow/.env b/480T/ragflow/.env new file mode 100755 index 0000000..02735ad --- /dev/null +++ b/480T/ragflow/.env @@ -0,0 +1,348 @@ +# ----------------------------------------------------------------------------- +# SECURITY WARNING: DO NOT DEPLOY WITH DEFAULT PASSWORDS +# For non-local deployments, please change all passwords (ELASTIC_PASSWORD, +# MYSQL_PASSWORD, MINIO_PASSWORD, etc.) to strong, unique values. +# You can generate a random string using: openssl rand -hex 32 +# ----------------------------------------------------------------------------- + +# ------------------------------ +# docker env var for specifying vector db type at startup +# (based on the vector db type, the corresponding docker +# compose profile will be used) +# ------------------------------ +# The type of doc engine to use. +# Available options: +# - `elasticsearch` (default) +# - `infinity` (https://github.com/infiniflow/infinity) +# - `oceanbase` (https://github.com/oceanbase/oceanbase) +# - `opensearch` (https://github.com/opensearch-project/OpenSearch) +# - `seekdb` (https://github.com/oceanbase/seekdb) +DOC_ENGINE=${DOC_ENGINE:-elasticsearch} + +# Device on which deepdoc inference run. +# Available levels: +# - `cpu` (default) +# - `gpu` +DEVICE=${DEVICE:-cpu} + +COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE} + +# The version of Elasticsearch. +STACK_VERSION=${STACK_VERSION:-8.11.3} + +# The hostname where the Elasticsearch service is exposed +ES_HOST=es01 + +# The port used to expose the Elasticsearch service to the host machine, +# allowing EXTERNAL access to the service running inside the Docker container. +ES_PORT=1200 + +# The password for Elasticsearch. +# WARNING: Change this for production! +ELASTIC_PASSWORD=infini_rag_flow + +# the hostname where OpenSearch service is exposed, set it not the same as elasticsearch +OS_PORT=1201 + +# The hostname where the OpenSearch service is exposed +OS_HOST=opensearch01 + +# The password for OpenSearch. +# At least one uppercase letter, one lowercase letter, one digit, and one special character +OPENSEARCH_PASSWORD=infini_rag_flow_OS_01 + +# The port used to expose the Kibana service to the host machine, +# allowing EXTERNAL access to the service running inside the Docker container. +# To enable kibana, you need to: +# 1. Ensure that COMPOSE_PROFILES includes kibana, for example: COMPOSE_PROFILES=${COMPOSE_PROFILES},kibana +# 2. Comment out or delete the following configurations of the es service in docker-compose-base.yml: xpack.security.enabled、xpack.security.http.ssl.enabled、xpack.security.transport.ssl.enabled (for details: https://www.elastic.co/docs/deploy-manage/security/self-auto-setup#stack-existing-settings-detected) +# 3. Adjust the es.hosts in conf/service_config.yaml or docker/service_conf.yaml.template to 'https://localhost:1200' +# 4. After the startup is successful, in the es container, execute the command to generate the kibana token: `bin/elasticsearch-create-enrollment-token -s kibana`, then you can use kibana normally +KIBANA_PORT=6601 + +# The maximum amount of the memory, in bytes, that a specific Docker container can use while running. +# Update it according to the available memory in the host machine. +MEM_LIMIT=8073741824 + +# The hostname where the Infinity service is exposed +INFINITY_HOST=infinity + +# Port to expose Infinity API to the host +INFINITY_THRIFT_PORT=23817 +INFINITY_HTTP_PORT=23820 +INFINITY_PSQL_PORT=5432 + +# The hostname where the OceanBase service is exposed +OCEANBASE_HOST=oceanbase +# The port used to expose the OceanBase service +OCEANBASE_PORT=2881 +# The username for OceanBase +OCEANBASE_USER=root@ragflow +# The password for OceanBase +OCEANBASE_PASSWORD=infini_rag_flow +# The doc database of the OceanBase service to use +OCEANBASE_DOC_DBNAME=ragflow_doc + +# OceanBase container configuration +OB_CLUSTER_NAME=${OB_CLUSTER_NAME:-ragflow} +OB_TENANT_NAME=${OB_TENANT_NAME:-ragflow} +OB_SYS_PASSWORD=${OCEANBASE_PASSWORD:-infini_rag_flow} +OB_TENANT_PASSWORD=${OCEANBASE_PASSWORD:-infini_rag_flow} +OB_MEMORY_LIMIT=${OB_MEMORY_LIMIT:-10G} +OB_SYSTEM_MEMORY=${OB_SYSTEM_MEMORY:-2G} +OB_DATAFILE_SIZE=${OB_DATAFILE_SIZE:-20G} +OB_LOG_DISK_SIZE=${OB_LOG_DISK_SIZE:-20G} + +# The hostname where the SeekDB service is exposed +SEEKDB_HOST=seekdb +# The port used to expose the SeekDB service +SEEKDB_PORT=2881 +# The username for SeekDB +SEEKDB_USER=root +# The password for SeekDB +SEEKDB_PASSWORD=infini_rag_flow +# The doc database of the SeekDB service to use +SEEKDB_DOC_DBNAME=ragflow_doc +# SeekDB memory limit +SEEKDB_MEMORY_LIMIT=2G + +# The password for MySQL. +# WARNING: Change this for production! +MYSQL_PASSWORD=infini_rag_flow +# The hostname where the MySQL service is exposed +MYSQL_HOST=mysql +# The database of the MySQL service to use +MYSQL_DBNAME=rag_flow +# The port used to connect to MySQL from RAGFlow container. +# Change this if you use external MySQL. +MYSQL_PORT=3306 +# The port used to expose the MySQL service to the host machine, +# allowing EXTERNAL access to the MySQL database running inside the Docker container. +EXPOSE_MYSQL_PORT=3306 +# The maximum size of communication packets sent to the MySQL server +MYSQL_MAX_PACKET=1073741824 + +# The hostname where the MinIO service is exposed +MINIO_HOST=minio +# The port used to expose the MinIO console interface to the host machine, +# allowing EXTERNAL access to the web-based console running inside the Docker container. +MINIO_CONSOLE_PORT=9001 +# The port used to expose the MinIO API service to the host machine, +# allowing EXTERNAL access to the MinIO object storage service running inside the Docker container. +MINIO_PORT=9000 +# The username for MinIO. +# When updated, you must revise the `minio.user` entry in service_conf.yaml accordingly. +MINIO_USER=rag_flow +# The password for MinIO. +# When updated, you must revise the `minio.password` entry in service_conf.yaml accordingly. +MINIO_PASSWORD=infini_rag_flow + +# The hostname where the Redis service is exposed +REDIS_HOST=redis +# The port used to expose the Redis service to the host machine, +# allowing EXTERNAL access to the Redis service running inside the Docker container. +REDIS_PORT=6379 +# The password for Redis. +REDIS_PASSWORD=infini_rag_flow + +NATS_HOST=nats +NATS_PORT=5222 + +# The hostname where the ClickHouse service is exposed +CLICKHOUSE_HOST=clickhouse +# The port used to expose the ClickHouse native TCP service +CLICKHOUSE_TCP_PORT=9900 +# The port used to expose the ClickHouse HTTP service +CLICKHOUSE_HTTP_PORT=8123 +# The username for ClickHouse +CLICKHOUSE_USER=ragflow +# The password for ClickHouse +CLICKHOUSE_PASSWORD=infini_rag_flow +# The database for ClickHouse +CLICKHOUSE_DATABASE=ragflow + +# Jaeger (distributed tracing) +# Enable by adding `jaeger` to COMPOSE_PROFILES, e.g.: +# COMPOSE_PROFILES=${COMPOSE_PROFILES},jaeger +# Then update otel.host in service_conf.yaml to "jaeger". +JAEGER_VERSION=2.19.0 +JAEGER_OTLP_GRPC_PORT=4317 +JAEGER_OTLP_HTTP_PORT=4318 +JAEGER_UI_PORT=16686 + +# The port used to expose RAGFlow's HTTP API service to the host machine, +# allowing EXTERNAL access to the service running inside the Docker container. +SVR_WEB_HTTP_PORT=80 +SVR_WEB_HTTPS_PORT=443 +SVR_HTTP_PORT=3600 +ADMIN_SVR_HTTP_PORT=9381 +SVR_MCP_PORT=9382 +GO_HTTP_PORT=9384 +GO_ADMIN_PORT=9383 + +# API_PROXY_SCHEME=hybrid # go and python hybrid deploy mode +# API_PROXY_SCHEME=go # use go server deployment +API_PROXY_SCHEME=python # use pure python server deployment + +# Development-only: set to 1 to bypass host safety checks for test_db_connection and allow private/local database hosts. +# Do not enable in production. +ALLOW_ANY_HOST=0 + +# The RAGFlow Docker image to download. v0.22+ doesn't include embedding models. +RAGFLOW_IMAGE=infiniflow/ragflow:v0.26.4 + +# If you cannot download the RAGFlow Docker image: +# RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:v0.26.4 +# RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:v0.26.4 +# +# - For the `nightly` edition, uncomment either of the following: +# RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:nightly +# RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:nightly + +# The embedding service image, model and port. +# Important: To enable the embedding service, you need to uncomment one of the following two lines: +# COMPOSE_PROFILES=${COMPOSE_PROFILES},tei-cpu +# COMPOSE_PROFILES=${COMPOSE_PROFILES},tei-gpu + +# The embedding service image: +TEI_IMAGE_CPU=infiniflow/text-embeddings-inference:cpu-1.8 +TEI_IMAGE_GPU=infiniflow/text-embeddings-inference:1.8 + +# The embedding service model: +# Available options: +# - `Qwen/Qwen3-Embedding-0.6B` (default, requires 25GB RAM/vRAM to load) +# - `BAAI/bge-m3` (requires 21GB RAM/vRAM to load) +# - `BAAI/bge-small-en-v1.5` (requires 1.2GB RAM/vRAM to load) +TEI_MODEL=${TEI_MODEL:-Qwen/Qwen3-Embedding-0.6B} + +# The embedding service port: +TEI_HOST=tei +# The port used to expose the TEI service to the host machine, +# allowing EXTERNAL access to the service running inside the Docker container. +TEI_PORT=6380 + +# The local time zone. +TZ=Asia/Shanghai + +# Uncomment the following line if you have limited access to huggingface.co: +# HF_ENDPOINT=https://hf-mirror.com + +# Optimizations for MacOS +# Uncomment the following line if your operating system is MacOS: +# MACOS=1 + +# The maximum file size limit (in bytes) for each upload to your dataset or RAGFlow's File system. +# To change the 1GB file size limit, uncomment the line below and update as needed. +# MAX_CONTENT_LENGTH=1073741824 +# After updating, ensure `client_max_body_size` in nginx/nginx.conf is updated accordingly. +# Note that neither `MAX_CONTENT_LENGTH` nor `client_max_body_size` sets the maximum size for files uploaded to an agent. +# See https://ragflow.io/docs/dev/begin_component for details. + +# Controls how many documents are processed in a single batch. +# Defaults to 4 if DOC_BULK_SIZE is not explicitly set. +DOC_BULK_SIZE=${DOC_BULK_SIZE:-4} + +# Defines the number of items to process per batch when generating embeddings. +# Defaults to 16 if EMBEDDING_BATCH_SIZE is not set in the environment. +EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-16} + +# Log level for the RAGFlow's own and imported packages. +# Available levels: +# - `DEBUG` +# - `INFO` (default) +# - `WARNING` +# - `ERROR` +# For example, the following line changes the log level of `ragflow.es_conn` to `DEBUG`: +# LOG_LEVELS=ragflow.es_conn=DEBUG + +# aliyun OSS configuration +# STORAGE_IMPL=OSS +# ACCESS_KEY=xxx +# SECRET_KEY=eee +# ENDPOINT=http://oss-cn-hangzhou.aliyuncs.com +# REGION=cn-hangzhou +# BUCKET=ragflow65536 +# + +# A user registration switch: +# - Enable registration: 1 +# - Disable registration: 0 +REGISTER_ENABLED=1 + +# ----------------------------------------------------------------------------- +# Sandbox +# ----------------------------------------------------------------------------- +# Sandbox provider type and runtime settings are configured in Admin > Sandbox +# Settings. + +# Enable sandbox support. +# SANDBOX_ENABLED=1 +# COMPOSE_PROFILES=${COMPOSE_PROFILES},sandbox + +# Shared sandbox settings +# The MinIO bucket name for storing sandbox-generated artifacts. +# SANDBOX_ARTIFACT_BUCKET=sandbox-artifacts + +# Number of days before sandbox artifacts are automatically deleted. +# SANDBOX_ARTIFACT_EXPIRE_DAYS=7 + +# Self-managed deployment defaults +# These values are used by the `sandbox` compose profile and shown in Admin as +# deployment defaults for the self-managed provider. +# Pull the required base images before running: +# docker pull infiniflow/sandbox-base-nodejs:latest +# docker pull infiniflow/sandbox-base-python:latest +# Default runtime images include: +# - Node.js base image: axios +# - Python base image: requests, numpy, pandas +# SANDBOX_EXECUTOR_MANAGER_IMAGE=${SANDBOX_EXECUTOR_MANAGER_IMAGE:-infiniflow/sandbox-executor-manager:latest} +# SANDBOX_EXECUTOR_MANAGER_POOL_SIZE=${SANDBOX_EXECUTOR_MANAGER_POOL_SIZE:-3} +# SANDBOX_BASE_PYTHON_IMAGE=${SANDBOX_BASE_PYTHON_IMAGE:-infiniflow/sandbox-base-python:latest} +# SANDBOX_BASE_NODEJS_IMAGE=${SANDBOX_BASE_NODEJS_IMAGE:-infiniflow/sandbox-base-nodejs:latest} +# SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT:-9385} +# SANDBOX_ENABLE_SECCOMP=false +# SANDBOX_MAX_MEMORY=256m # b, k, m, g +# SANDBOX_TIMEOUT=10s # s, m, 1m30s + +# ----------------------------------------------------------------------------- +# Sandbox End +# ----------------------------------------------------------------------------- + +# Enable DocLing +USE_DOCLING=false + +# Enable Mineru +# Uncommenting these lines will automatically add MinerU to the model provider whenever possible. +# More details see https://ragflow.io/docs/faq#how-to-use-mineru-to-parse-pdf-documents. +# MINERU_DELETE_OUTPUT=0 # keep output directory +# MINERU_BACKEND=pipeline # or another backend you prefer + +# pptx support +DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 + +# crypto utils +# RAGFLOW_CRYPTO_ENABLED=true +# RAGFLOW_CRYPTO_ALGORITHM=aes-256-cbc # one of aes-256-cbc, aes-128-cbc, sm4-cbc +# RAGFLOW_CRYPTO_KEY=ragflow-crypto-key + + +# Used for ThreadPoolExecutor +THREAD_POOL_MAX_WORKERS=128 + +#Option to disable login form for SSO +DISABLE_PASSWORD_LOGIN=false + +# ----------------------------------------------------------------------------- +# DeepDoc OSS Vision Service +# ----------------------------------------------------------------------------- +# URL for the deepdoc vision API (DLA, OCR, TSR) served by OSS ONNX models. +# The `deepdoc` service defined in docker-compose.yml provides this endpoint. +# When unset, the parser falls back to inline ONNX Runtime inference. + +# Comment existing COMPOSE_PROFILES and uncomment below if need deepdoc service. +# COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE},deepdoc + +# DEEPDOC_URL=http://deepdoc:9390 + +# Docker image for the OSS deepdoc service. CPU-only; uses ONNX Runtime. +# DEEPDOC_IMAGE=deepdoc_oss:latest diff --git a/480T/ragflow/.env.single-bucket-example b/480T/ragflow/.env.single-bucket-example new file mode 100755 index 0000000..96120b3 --- /dev/null +++ b/480T/ragflow/.env.single-bucket-example @@ -0,0 +1,108 @@ +# Example: Single Bucket Mode Configuration +# +# This file shows how to configure RAGFlow to use a single MinIO/S3 bucket +# with directory structure instead of creating multiple buckets. + +# ============================================================================ +# MinIO/S3 Configuration for Single Bucket Mode +# ============================================================================ + +# MinIO/S3 Endpoint (with port if not default) +# For HTTPS (port 443), the connection will automatically use secure=True +export MINIO_HOST=minio.example.com:443 + +# Access credentials +export MINIO_USER=your-access-key +export MINIO_PASSWORD=your-secret-password-here + +# Single Bucket Configuration (NEW!) +# If set, all data will be stored in this bucket instead of creating +# separate buckets for each knowledge base +export MINIO_BUCKET=ragflow-bucket + +# Optional: Prefix path within the bucket (NEW!) +# If set, all files will be stored under this prefix +# Example: bucket/prefix_path/kb_id/file.pdf +export MINIO_PREFIX_PATH=ragflow + +# ============================================================================ +# Alternative: Multi-Bucket Mode (Default) +# ============================================================================ +# +# To use the original multi-bucket mode, simply don't set MINIO_BUCKET +# and MINIO_PREFIX_PATH: +# +# export MINIO_HOST=minio.local +# export MINIO_USER=admin +# export MINIO_PASSWORD=password +# # MINIO_BUCKET not set +# # MINIO_PREFIX_PATH not set + +# ============================================================================ +# Storage Mode Selection (Environment Variable) +# ============================================================================ +# +# Make sure this is set to use MinIO (default) +export STORAGE_IMPL=MINIO + +# ============================================================================ +# Example Path Structures +# ============================================================================ +# +# Multi-Bucket Mode (default): +# bucket: kb_12345/file.pdf +# bucket: kb_67890/file.pdf +# bucket: folder_abc/file.txt +# +# Single Bucket Mode (MINIO_BUCKET set): +# bucket: ragflow-bucket/kb_12345/file.pdf +# bucket: ragflow-bucket/kb_67890/file.pdf +# bucket: ragflow-bucket/folder_abc/file.txt +# +# Single Bucket with Prefix (both set): +# bucket: ragflow-bucket/ragflow/kb_12345/file.pdf +# bucket: ragflow-bucket/ragflow/kb_67890/file.pdf +# bucket: ragflow-bucket/ragflow/folder_abc/file.txt + +# ============================================================================ +# IAM Policy for Single Bucket Mode +# ============================================================================ +# +# When using single bucket mode, you only need permissions for one bucket: +# +# { +# "Version": "2012-10-17", +# "Statement": [ +# { +# "Effect": "Allow", +# "Action": ["s3:*"], +# "Resource": [ +# "arn:aws:s3:::ragflow-bucket", +# "arn:aws:s3:::ragflow-bucket/*" +# ] +# } +# ] +# } + +# ============================================================================ +# Testing the Configuration +# ============================================================================ +# +# After setting these variables, you can test with MinIO Client (mc): +# +# # Configure mc alias +# mc alias set ragflow https://minio.example.com:443 \ +# your-access-key \ +# your-secret-password-here +# +# # List bucket contents +# mc ls ragflow/ragflow-bucket/ +# +# # If prefix is set, check the prefix +# mc ls ragflow/ragflow-bucket/ragflow/ +# +# # Test write permission +# echo "test" | mc pipe ragflow/ragflow-bucket/ragflow/_test.txt +# +# # Clean up test file +# mc rm ragflow/ragflow-bucket/ragflow/_test.txt diff --git a/480T/ragflow/README.md b/480T/ragflow/README.md new file mode 100755 index 0000000..b5cdb13 --- /dev/null +++ b/480T/ragflow/README.md @@ -0,0 +1,329 @@ +# README + +
+📗 Table of Contents + +- 🐳 [Docker Compose](#-docker-compose) +- 🐬 [Docker environment variables](#-docker-environment-variables) +- 🐋 [Service configuration](#-service-configuration) +- 📋 [Setup Examples](#-setup-examples) + +
+ +## 🐳 Docker Compose + +- **docker-compose.yml** + Sets up environment for RAGFlow and its dependencies. +- **docker-compose-base.yml** + Sets up environment for RAGFlow's dependencies: Elasticsearch/[Infinity](https://github.com/infiniflow/infinity), MySQL, MinIO, and Redis. + +> [!CAUTION] +> We do not actively maintain **docker-compose-CN-oc9.yml**, **docker-compose-macos.yml**, so use them at your own risk. However, you are welcome to file a pull request to improve any of them. + +## 🐬 Docker environment variables + +The [.env](./.env) file contains important environment variables for Docker. + +### Elasticsearch + +- `STACK_VERSION` + The version of Elasticsearch. Defaults to `8.11.3` +- `ES_PORT` + The port used to expose the Elasticsearch service to the host machine, allowing **external** access to the service running inside the Docker container. Defaults to `1200`. +- `ELASTIC_PASSWORD` + The password for Elasticsearch. + +### Kibana + +- `KIBANA_PORT` + The port used to expose the Kibana service to the host machine, allowing **external** access to the service running inside the Docker container. Defaults to `6601`. +- `KIBANA_USER` + The username for Kibana. Defaults to `rag_flow`. +- `KIBANA_PASSWORD` + The password for Kibana. Defaults to `infini_rag_flow`. + +### Resource management + +- `MEM_LIMIT` + The maximum amount of the memory, in bytes, that *a specific* Docker container can use while running. Defaults to `8073741824`. + +### MySQL + +- `MYSQL_PASSWORD` + The password for MySQL. +- `MYSQL_PORT` + The port to connect to MySQL from RAGFlow container. Defaults to `3306`. Change this if you use an external MySQL. +- `EXPOSE_MYSQL_PORT` + The port used to expose the MySQL service to the host machine, allowing **external** access to the MySQL database running inside the Docker container. Defaults to `5455`. + +### MinIO + +- `MINIO_CONSOLE_PORT` + The port used to expose the MinIO console interface to the host machine, allowing **external** access to the web-based console running inside the Docker container. Defaults to `9001` +- `MINIO_PORT` + The port used to expose the MinIO API service to the host machine, allowing **external** access to the MinIO object storage service running inside the Docker container. Defaults to `9000`. +- `MINIO_USER` + The username for MinIO. +- `MINIO_PASSWORD` + The password for MinIO. + +### Redis + +- `REDIS_PORT` + The port used to expose the Redis service to the host machine, allowing **external** access to the Redis service running inside the Docker container. Defaults to `6379`. +- `REDIS_PASSWORD` + The password for Redis. + +### RAGFlow + +- `SVR_HTTP_PORT` + The port used to expose RAGFlow's HTTP API service to the host machine, allowing **external** access to the service running inside the Docker container. Defaults to `9380`. +- `RAGFLOW_IMAGE` + The Docker image edition. Defaults to `infiniflow/ragflow:v0.26.4`. The RAGFlow Docker image does not include embedding models. + + +> [!TIP] +> If you cannot download the RAGFlow Docker image, try the following mirrors. +> +> - For the `nightly` edition: +> - `RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:nightly` or, +> - `RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:nightly`. + +### DeepDoc Vision Service (OSS) + +- `DEEPDOC_URL` + URL for the deepdoc vision API serving DLA (layout analysis), OCR (text detection/recognition), and TSR (table structure recognition). The `deepdoc` service in `docker-compose.yml` provides this endpoint. Defaults to `http://deepdoc:9390`. When unset, the parser falls back to inline ONNX Runtime inference. + + > The OSS deepdoc service runs on CPU using ONNX Runtime models. No GPU required. + > API endpoints: `GET /health`, `GET /model`, `POST /predict/dla`, `POST /predict/tsr`, `POST /predict/ocr`. + +- `DEEPDOC_IMAGE` + Docker image for the OSS deepdoc service. Defaults to `infiniflow/deepdoc_oss:latest`. + +### Timezone + +- `TZ` + The local time zone. Defaults to `'Asia/Shanghai'`. + +### Hugging Face mirror site + +- `HF_ENDPOINT` + The mirror site for huggingface.co. It is disabled by default. You can uncomment this line if you have limited access to the primary Hugging Face domain. + +### MacOS + +- `MACOS` + Optimizations for macOS. It is disabled by default. You can uncomment this line if your OS is macOS. + +### Maximum file size + +- `MAX_CONTENT_LENGTH` + The maximum file size for each uploaded file, in bytes. You can uncomment this line if you wish to change the 128M file size limit. After making the change, ensure you update `client_max_body_size` in nginx/nginx.conf correspondingly. + +### Doc bulk size + +- `DOC_BULK_SIZE` + The number of document chunks processed in a single batch during document parsing. Defaults to `4`. + +### Embedding batch size + +- `EMBEDDING_BATCH_SIZE` + The number of text chunks processed in a single batch during embedding vectorization. Defaults to `16`. + +### OceanBase prerequisites + +Before setting `DOC_ENGINE=oceanbase`, make sure the host OS allows the file descriptor and core dump limits OceanBase expects. + +1. Set host limits: + + ```bash + sudo tee /etc/security/limits.d/99-oceanbase.conf >/dev/null <<'EOF' + root soft nofile 655350 + root hard nofile 655350 + * soft nofile 655350 + * hard nofile 655350 + * soft core unlimited + * hard core unlimited + EOF + ``` + +2. Make sure PAM limits are enabled: + + ```bash + grep -E 'pam_limits\.so' /etc/pam.d/common-session /etc/pam.d/common-session-noninteractive + ``` + + If missing, add them: + + ```bash + echo 'session required pam_limits.so' | sudo tee -a /etc/pam.d/common-session + echo 'session required pam_limits.so' | sudo tee -a /etc/pam.d/common-session-noninteractive + ``` + +3. Log out and log back in, or reboot. + +4. Verify the effective limit: + + ```bash + ulimit -n + ``` + + Expected: `655350`, or at least `20000`. + +## 🐋 Service configuration + +[service_conf.yaml](./service_conf.yaml) specifies the system-level configuration for RAGFlow and is used by its API server and task executor. In a dockerized setup, this file is automatically created based on the [service_conf.yaml.template](./service_conf.yaml.template) file (replacing all environment variables by their values). + +- `ragflow` + - `host`: The API server's IP address inside the Docker container. Defaults to `0.0.0.0`. + - `port`: The API server's serving port inside the Docker container. Defaults to `9380`. + +- `deepdoc` + The OSS DeepDoc vision service provides DLA, OCR, and TSR inference via ONNX Runtime. + Defined in `docker-compose.yml`, it is started automatically as a dependency of `ragflow-cpu` and `ragflow-gpu`. + - `image`: Docker image. Defaults to `infiniflow/deepdoc_oss:latest`. + - `port`: Serving port inside the container. Defaults to `9390`. + - Health check: `curl -f http://localhost:9390/health` every 10s. + +- `mysql` + - `name`: The MySQL database name. Defaults to `rag_flow`. + - `user`: The username for MySQL. + - `password`: The password for MySQL. + - `port`: The MySQL serving port inside the Docker container. Defaults to `3306`. + - `max_connections`: The maximum number of concurrent connections to the MySQL database. Defaults to `100`. + - `stale_timeout`: Timeout in seconds. + +- `minio` + - `user`: The username for MinIO. + - `password`: The password for MinIO. + - `host`: The MinIO serving IP *and* port inside the Docker container. Defaults to `minio:9000`. + +- `oceanbase` + - `scheme`: The connection scheme. Set to `mysql` to use mysql config, or other values to use config below. + - `config`: + - `db_name`: The OceanBase database name. + - `user`: The username for OceanBase. + - `password`: The password for OceanBase. + - `host`: The hostname of the OceanBase service. + - `port`: The port of OceanBase. + +- `oss` + - `access_key`: The access key ID used to authenticate requests to the OSS service. + - `secret_key`: The secret access key used to authenticate requests to the OSS service. + - `endpoint_url`: The URL of the OSS service endpoint. + - `region`: The OSS region where the bucket is located. + - `bucket`: The name of the OSS bucket where files will be stored. When you want to store all files in a specified bucket, you need this configuration item. + - `prefix_path`: Optional. A prefix path to prepend to file names in the OSS bucket, which can help organize files within the bucket. + +- `s3`: + - `access_key`: The access key ID used to authenticate requests to the S3 service. + - `secret_key`: The secret access key used to authenticate requests to the S3 service. + - `endpoint_url`: The URL of the S3-compatible service endpoint. This is necessary when using an S3-compatible protocol instead of the default AWS S3 endpoint. + - `bucket`: The name of the S3 bucket where files will be stored. When you want to store all files in a specified bucket, you need this configuration item. + - `region`: The AWS region where the S3 bucket is located. This is important for directing requests to the correct data center. + - `signature_version`: Optional. The version of the signature to use for authenticating requests. Common versions include `v4`. + - `addressing_style`: Optional. The style of addressing to use for the S3 endpoint. This can be `path` or `virtual`. + - `prefix_path`: Optional. A prefix path to prepend to file names in the S3 bucket, which can help organize files within the bucket. + +- `oauth` + The OAuth configuration for signing up or signing in to RAGFlow using a third-party account. + - ``: Custom channel ID. + - `type`: Authentication type, options include `oauth2`, `oidc`, `github`. Default is `oauth2`, when `issuer` parameter is provided, defaults to `oidc`. + - `icon`: Icon ID, options include `github`, `sso`, default is `sso`. + - `display_name`: Channel name, defaults to the Title Case format of the channel ID. + - `client_id`: Required, unique identifier assigned to the client application. + - `client_secret`: Required, secret key for the client application, used for communication with the authentication server. + - `authorization_url`: Base URL for obtaining user authorization. + - `token_url`: URL for exchanging authorization code and obtaining access token. + - `userinfo_url`: URL for obtaining user information (username, email, etc.). + - `issuer`: Base URL of the identity provider. OIDC clients can dynamically obtain the identity provider's metadata (`authorization_url`, `token_url`, `userinfo_url`) through `issuer`. + - `scope`: Requested permission scope, a space-separated string. For example, `openid profile email`. + - `redirect_uri`: Required, URI to which the authorization server redirects during the authentication flow to return results. Must match the callback URI registered with the authentication server. Format: `https://your-app.com/v1/user/oauth/callback/`. For local configuration, you can directly use `http://127.0.0.1:80/v1/user/oauth/callback/`. + +- `user_default_llm` + The default LLM to use for a new RAGFlow user. It is disabled by default. To enable this feature, uncomment the corresponding lines in **service_conf.yaml.template**. + - `factory`: The LLM supplier. Available options: + - `"OpenAI"` + - `"DeepSeek"` + - `"Moonshot"` + - `"Tongyi-Qianwen"` + - `"VolcEngine"` + - `"ZHIPU-AI"` + - `api_key`: The API key for the specified LLM. You will need to apply for your model API key online. + +> [!TIP] +> If you do not set the default LLM here, configure the default LLM on the **Settings** page in the RAGFlow UI. + + +## 📋 Setup Examples + +### 🔒 HTTPS Setup + +#### Prerequisites + +- A registered domain name pointing to your server +- Port 80 and 443 open on your server +- Docker and Docker Compose installed + +#### Getting and configuring certificates (Let's Encrypt) + +If you want your instance to be available under `https`, follow these steps: + +1. **Install Certbot and obtain certificates** + ```bash + # Ubuntu/Debian + sudo apt update && sudo apt install certbot + + # CentOS/RHEL + sudo yum install certbot + + # Obtain certificates (replace with your actual domain) + sudo certbot certonly --standalone -d your-ragflow-domain.com + ``` + +2. **Locate your certificates** + Once generated, your certificates will be located at: + - Certificate: `/etc/letsencrypt/live/your-ragflow-domain.com/fullchain.pem` + - Private key: `/etc/letsencrypt/live/your-ragflow-domain.com/privkey.pem` + +3. **Update docker-compose.yml** + Add the certificate volumes to the `ragflow` service in your `docker-compose.yml`: + ```yaml + services: + ragflow: + # ...existing configuration... + volumes: + # SSL certificates + - /etc/letsencrypt/live/your-ragflow-domain.com/fullchain.pem:/etc/nginx/ssl/fullchain.pem:ro + - /etc/letsencrypt/live/your-ragflow-domain.com/privkey.pem:/etc/nginx/ssl/privkey.pem:ro + # Switch to HTTPS nginx configuration + - ./nginx/ragflow.https.conf:/etc/nginx/conf.d/ragflow.conf + # ...other existing volumes... + + ``` + +4. **Update nginx configuration** + Edit `nginx/ragflow.https.conf` and replace `my_ragflow_domain.com` with your actual domain name. + +5. **Restart the services** + ```bash + docker-compose down + docker-compose up -d + ``` + + +> [!IMPORTANT] +> - Ensure your domain's DNS A record points to your server's IP address +> - Stop any services running on ports 80/443 before obtaining certificates with `--standalone` + +> [!TIP] +> For development or testing, you can use self-signed certificates, but browsers will show security warnings. + +#### Alternative: Using existing certificates + +If you already have SSL certificates from another provider: + +1. Place your certificates in a directory accessible to Docker +2. Update the volume paths in `docker-compose.yml` to point to your certificate files +3. Ensure the certificate file contains the full certificate chain +4. Follow steps 4-5 from the Let's Encrypt guide above diff --git a/480T/ragflow/docker-compose-base.yml b/480T/ragflow/docker-compose-base.yml new file mode 100755 index 0000000..1afffe7 --- /dev/null +++ b/480T/ragflow/docker-compose-base.yml @@ -0,0 +1,404 @@ +services: + es01: + profiles: + - elasticsearch + image: elasticsearch:${STACK_VERSION} + volumes: + - esdata01:/usr/share/elasticsearch/data + # Official ES image ACL on /tmp denies writes for user elasticsearch (r-x only). + # entrypoint.sh needs a writable temp dir for bash here-documents. + tmpfs: + - /tmp:mode=1777,size=512m + ports: + - ${ES_PORT}:9200 + env_file: .env + environment: + - node.name=es01 + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=false + - discovery.type=single-node + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=false + - xpack.security.transport.ssl.enabled=false + - cluster.routing.allocation.disk.watermark.low=5gb + - cluster.routing.allocation.disk.watermark.high=3gb + - cluster.routing.allocation.disk.watermark.flood_stage=2gb + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: ["CMD-SHELL", "curl http://localhost:9200"] + interval: 10s + timeout: 10s + retries: 120 + networks: + - ragflow + restart: unless-stopped + + opensearch01: + profiles: + - opensearch + image: opensearchproject/opensearch:2.19.1 + volumes: + - osdata01:/usr/share/opensearch/data + ports: + - ${OS_PORT}:9201 + env_file: .env + environment: + - node.name=opensearch01 + - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD} + - bootstrap.memory_lock=false + - discovery.type=single-node + - plugins.security.disabled=false + - plugins.security.ssl.http.enabled=false + - plugins.security.ssl.transport.enabled=true + - cluster.routing.allocation.disk.watermark.low=5gb + - cluster.routing.allocation.disk.watermark.high=3gb + - cluster.routing.allocation.disk.watermark.flood_stage=2gb + - http.port=9201 + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: ["CMD-SHELL", "curl http://localhost:9201"] + interval: 10s + timeout: 10s + retries: 120 + networks: + - ragflow + restart: unless-stopped + + infinity: + profiles: + - infinity + image: infiniflow/infinity:v0.7.2-x64-v3 + volumes: + - infinity_data:/var/infinity + - ./infinity_conf.toml:/infinity_conf.toml + command: ["-f", "/infinity_conf.toml"] + ports: + - ${INFINITY_THRIFT_PORT}:23817 + - ${INFINITY_HTTP_PORT}:23820 + - ${INFINITY_PSQL_PORT}:5432 + env_file: .env + mem_limit: ${MEM_LIMIT} + ulimits: + nofile: + soft: 500000 + hard: 500000 + networks: + - ragflow + healthcheck: + test: ["CMD", "curl", "http://localhost:23820/admin/node/current"] + interval: 10s + timeout: 10s + retries: 120 + restart: unless-stopped + + oceanbase: + profiles: + - oceanbase + image: oceanbase/oceanbase-ce:4.4.1.0-100000032025101610 + entrypoint: ["bash", "/root/boot/ragflow-oceanbase-entrypoint.sh"] + ulimits: + nofile: + soft: 655350 + hard: 655350 + volumes: + - ./oceanbase/data:/root/ob + - ./oceanbase/conf:/root/.obd/cluster + - ./oceanbase/init.d:/root/boot/init.d + - ./oceanbase-entrypoint.sh:/root/boot/ragflow-oceanbase-entrypoint.sh:ro + ports: + - ${OCEANBASE_PORT:-2881}:2881 + env_file: .env + environment: + - MODE=normal + - OB_SERVER_IP=127.0.0.1 + mem_limit: ${MEM_LIMIT} + healthcheck: + test: [ 'CMD-SHELL', 'obclient -h127.0.0.1 -P2881 -uroot@${OB_TENANT_NAME:-ragflow} -p${OB_TENANT_PASSWORD:-infini_rag_flow} -e "CREATE DATABASE IF NOT EXISTS ${OCEANBASE_DOC_DBNAME:-ragflow_doc};"' ] + interval: 10s + retries: 30 + start_period: 30s + timeout: 10s + networks: + - ragflow + restart: unless-stopped + + seekdb: + profiles: + - seekdb + image: oceanbase/seekdb:latest + container_name: seekdb + volumes: + - ./seekdb:/var/lib/oceanbase + ports: + - ${SEEKDB_PORT:-2881}:2881 + env_file: .env + environment: + - ROOT_PASSWORD=${SEEKDB_PASSWORD:-infini_rag_flow} + - MEMORY_LIMIT=${SEEKDB_MEMORY_LIMIT:-2G} + - REPORTER=ragflow-seekdb + mem_limit: ${MEM_LIMIT} + healthcheck: + test: ['CMD-SHELL', 'mysql -h127.0.0.1 -P2881 -uroot -p${SEEKDB_PASSWORD:-infini_rag_flow} -e "CREATE DATABASE IF NOT EXISTS ${SEEKDB_DOC_DBNAME:-ragflow_doc};"'] + interval: 5s + retries: 60 + timeout: 5s + networks: + - ragflow + restart: unless-stopped + + sandbox-executor-manager: + profiles: + - sandbox + image: ${SANDBOX_EXECUTOR_MANAGER_IMAGE-infiniflow/sandbox-executor-manager:latest} + privileged: true + ports: + - ${SANDBOX_EXECUTOR_MANAGER_PORT-9385}:9385 + env_file: .env + volumes: + - /var/run/docker.sock:/var/run/docker.sock + networks: + - ragflow + security_opt: + - no-new-privileges:true + environment: + - SANDBOX_EXECUTOR_MANAGER_POOL_SIZE=${SANDBOX_EXECUTOR_MANAGER_POOL_SIZE:-3} + - SANDBOX_BASE_PYTHON_IMAGE=${SANDBOX_BASE_PYTHON_IMAGE:-infiniflow/sandbox-base-python:latest} + - SANDBOX_BASE_NODEJS_IMAGE=${SANDBOX_BASE_NODEJS_IMAGE:-infiniflow/sandbox-base-nodejs:latest} + - SANDBOX_ENABLE_SECCOMP=${SANDBOX_ENABLE_SECCOMP:-false} + - SANDBOX_MAX_MEMORY=${SANDBOX_MAX_MEMORY:-256m} + - SANDBOX_TIMEOUT=${SANDBOX_TIMEOUT:-10s} + healthcheck: + test: ["CMD", "curl", "http://localhost:9385/healthz"] + interval: 10s + timeout: 10s + retries: 120 + restart: unless-stopped + + mysql: + # mysql:5.7 linux/arm64 image is unavailable. + image: mysql:8.0.40 + env_file: .env + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_PASSWORD} + command: + --max_connections=1000 + --character-set-server=utf8mb4 + --collation-server=utf8mb4_unicode_ci + --default-authentication-plugin=mysql_native_password + --tls_version="TLSv1.2,TLSv1.3" + --init-file /data/application/init.sql + --binlog_expire_logs_seconds=604800 + ports: + - ${EXPOSE_MYSQL_PORT}:3306 + volumes: + - mysql_data:/var/lib/mysql + - ./init.sql:/data/application/init.sql + networks: + - ragflow + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-uroot", "-p${MYSQL_PASSWORD}"] + interval: 10s + timeout: 10s + retries: 120 + restart: unless-stopped + + minio: + image: pgsty/minio:RELEASE.2026-03-25T00-00-00Z + command: ["server", "--console-address", ":9001", "/data"] + ports: + - ${MINIO_PORT}:9000 + - ${MINIO_CONSOLE_PORT}:9001 + env_file: .env + environment: + - MINIO_ROOT_USER=${MINIO_USER} + - MINIO_ROOT_PASSWORD=${MINIO_PASSWORD} + volumes: + - minio_data:/data + networks: + - ragflow + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 10s + retries: 120 + + redis: + # swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/valkey/valkey:8 + image: valkey/valkey:8 + command: ["redis-server", "--requirepass", "${REDIS_PASSWORD}", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru"] + env_file: .env + ports: + - ${REDIS_PORT}:6379 + volumes: + - redis_data:/data + networks: + - ragflow + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] + interval: 10s + timeout: 10s + retries: 120 + + jaeger: + profiles: + - jaeger + image: jaegertracing/jaeger:${JAEGER_VERSION:-2.19.0} + ports: + - ${JAEGER_OTLP_GRPC_PORT:-4317}:4317 + - ${JAEGER_OTLP_HTTP_PORT:-4318}:4318 + - ${JAEGER_UI_PORT:-16686}:16686 + env_file: .env + environment: + - COLLECTOR_OTLP_ENABLED=true + networks: + - ragflow + restart: unless-stopped + + nats: + profiles: + - ragflow-go + image: nats:2.14.2 + ports: + - ${NATS_PORT:-4222}:4222 + - "8222:8222" + volumes: + - nats_data:/data + command: -js -sd /data + env_file: .env + networks: + - ragflow + restart: unless-stopped + healthcheck: + test: ["CMD", "nats-server", "--health-check"] + interval: 10s + timeout: 10s + retries: 30 + + tei-cpu: + profiles: + - tei-cpu + image: ${TEI_IMAGE_CPU} + hostname: tei + ports: + - ${TEI_PORT-6380}:80 + env_file: .env + networks: + - ragflow + command: ["--model-id", "/data/${TEI_MODEL}", "--auto-truncate"] + restart: unless-stopped + + + tei-gpu: + profiles: + - tei-gpu + image: ${TEI_IMAGE_GPU} + hostname: tei + ports: + - ${TEI_PORT-6380}:80 + env_file: .env + networks: + - ragflow + command: ["--model-id", "/data/${TEI_MODEL}", "--auto-truncate"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + restart: unless-stopped + + + kibana: + profiles: + - kibana + image: kibana:${STACK_VERSION} + ports: + - ${KIBANA_PORT-5601}:5601 + env_file: .env + volumes: + - kibana_data:/usr/share/kibana/data + depends_on: + es01: + condition: service_started + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5601/api/status"] + interval: 10s + timeout: 10s + retries: 120 + networks: + - ragflow + restart: unless-stopped + + + clickhouse: + profiles: + - clickhouse + image: clickhouse/clickhouse-server:26.5.5.8 + volumes: + - clickhouse_data:/var/lib/clickhouse + - ./init-clickhouse.sql:/docker-entrypoint-initdb.d/init.sql + ports: + - ${CLICKHOUSE_TCP_PORT:-9900}:9000 + - ${CLICKHOUSE_HTTP_PORT:-8123}:8123 + env_file: .env + environment: + - CLICKHOUSE_USER=${CLICKHOUSE_USER:-ragflow} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-infini_rag_flow} + - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + mem_limit: ${MEM_LIMIT} + ulimits: + nofile: + soft: 655350 + hard: 655350 + healthcheck: + test: ["CMD", "clickhouse-client", "--user", "${CLICKHOUSE_USER:-ragflow}", "--password", "${CLICKHOUSE_PASSWORD:-infini_rag_flow}", "-q", "SELECT 1"] + interval: 10s + timeout: 10s + retries: 30 + start_period: 30s + networks: + - ragflow + restart: unless-stopped + + +volumes: + esdata01: + driver: local + osdata01: + driver: local + infinity_data: + driver: local + ob_data: + driver: local + seekdb_data: + driver: local + mysql_data: + driver: local + minio_data: + driver: local + redis_data: + driver: local + tei_data: + driver: local + kibana_data: + driver: local + nats_data: + driver: local + clickhouse_data: + driver: local + +networks: + ragflow: + driver: bridge diff --git a/480T/ragflow/docker-compose.yaml b/480T/ragflow/docker-compose.yaml new file mode 100755 index 0000000..14415f8 --- /dev/null +++ b/480T/ragflow/docker-compose.yaml @@ -0,0 +1,151 @@ +include: + - ./docker-compose-base.yml +# To ensure that the container processes the locally modified `service_conf.yaml.template` instead of the one included in its image, you need to mount the local `service_conf.yaml.template` to the container. +services: + deepdoc: + image: ${DEEPDOC_IMAGE:-deepdoc_oss:latest} + profiles: + - deepdoc + build: + context: .. + dockerfile: Dockerfile_deepdoc_oss + networks: + - ragflow + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9390/health"] + interval: 10s + timeout: 10s + retries: 60 + + ragflow-cpu: + depends_on: + mysql: + condition: service_healthy + profiles: + - cpu + image: ${RAGFLOW_IMAGE} + # Example configuration to set up an MCP server: + # command: + # - --enable-mcpserver + # - --mcp-host=0.0.0.0 + # - --mcp-port=9382 + # - --mcp-base-url=http://127.0.0.1:9380 + # - --mcp-script-path=/ragflow/mcp/server/server.py + # - --mcp-mode=self-host + # - --mcp-host-api-key=ragflow-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + # Optional transport flags for MCP (customize if needed). + # Host mode need to combined with --no-transport-streamable-http-enabled flag, namely, host+streamable-http is not supported yet. + # The following are enabled by default unless explicitly disabled with --no-. + # - --no-transport-sse-enabled # Disable legacy SSE endpoints (/sse and /messages/) + # - --no-transport-streamable-http-enabled # Disable Streamable HTTP transport (/mcp endpoint) + # - --no-json-response # Disable JSON response mode in Streamable HTTP transport (instead of SSE over HTTP) + + # Example configuration to start Admin server: + command: + - --enable-adminserver + - --init-model-provider-tables + ports: + - ${SVR_WEB_HTTP_PORT}:80 + - ${SVR_WEB_HTTPS_PORT}:443 + - ${SVR_HTTP_PORT}:9380 + - ${ADMIN_SVR_HTTP_PORT}:9381 + - ${SVR_MCP_PORT}:9382 # entry for MCP (host_port:docker_port). The docker_port must match the value you set for `mcp-port` above. + - ${GO_HTTP_PORT}:9384 + - ${GO_ADMIN_PORT}:9383 + volumes: + - ./ragflow-logs:/ragflow/logs + # - ./nginx/ragflow.conf:/etc/nginx/conf.d/ragflow.conf + # - ./nginx/proxy.conf:/etc/nginx/proxy.conf + # - ./nginx/nginx.conf:/etc/nginx/nginx.conf + - ./service_conf.yaml.template:/ragflow/conf/service_conf.yaml.template + - ./entrypoint.sh:/ragflow/entrypoint.sh + env_file: .env + networks: + - ragflow + restart: unless-stopped + # https://docs.docker.com/engine/daemon/prometheus/#create-a-prometheus-configuration + # If you use Docker Desktop, the --add-host flag is optional. This flag ensures that the host's internal IP is exposed to the Prometheus container. + extra_hosts: + - "host.docker.internal:host-gateway" + + ragflow-gpu: + depends_on: + mysql: + condition: service_healthy + profiles: + - gpu + image: ${RAGFLOW_IMAGE} + # Example configuration to set up an MCP server: + # command: + # - --enable-mcpserver + # - --mcp-host=0.0.0.0 + # - --mcp-port=9382 + # - --mcp-base-url=http://127.0.0.1:9380 + # - --mcp-script-path=/ragflow/mcp/server/server.py + # - --mcp-mode=self-host + # - --mcp-host-api-key=ragflow-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + # Optional transport flags for MCP (customize if needed). + # Host mode need to combined with --no-transport-streamable-http-enabled flag, namely, host+streamable-http is not supported yet. + # The following are enabled by default unless explicitly disabled with --no-. + # - --no-transport-sse-enabled # Disable legacy SSE endpoints (/sse and /messages/) + # - --no-transport-streamable-http-enabled # Disable Streamable HTTP transport (/mcp endpoint) + # - --no-json-response # Disable JSON response mode in Streamable HTTP transport (instead of SSE over HTTP) + + # Example configuration to start Admin server: + command: + - --enable-adminserver + ports: + - ${SVR_WEB_HTTP_PORT}:80 + - ${SVR_WEB_HTTPS_PORT}:443 + - ${SVR_HTTP_PORT}:9380 + - ${ADMIN_SVR_HTTP_PORT}:9381 + - ${SVR_MCP_PORT}:9382 # entry for MCP (host_port:docker_port). The docker_port must match the value you set for `mcp-port` above. + volumes: + - ./ragflow-logs:/ragflow/logs + # - ./nginx/ragflow.conf:/etc/nginx/conf.d/ragflow.conf + # - ./nginx/proxy.conf:/etc/nginx/proxy.conf + # - ./nginx/nginx.conf:/etc/nginx/nginx.conf + - ./service_conf.yaml.template:/ragflow/conf/service_conf.yaml.template + - ./entrypoint.sh:/ragflow/entrypoint.sh + env_file: .env + networks: + - ragflow + restart: unless-stopped + # https://docs.docker.com/engine/daemon/prometheus/#create-a-prometheus-configuration + # If you use Docker Desktop, the --add-host flag is optional. This flag ensures that the host's internal IP is exposed to the Prometheus container. + extra_hosts: + - "host.docker.internal:host-gateway" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + + # executor: + # depends_on: + # mysql: + # condition: service_healthy + # image: ${RAGFLOW_IMAGE} + # volumes: + # - ./ragflow-logs:/ragflow/logs + # - ./nginx/ragflow.conf:/etc/nginx/conf.d/ragflow.conf + # env_file: .env + # entrypoint: "/ragflow/entrypoint_task_executor.sh 1 3" + # networks: + # - ragflow + # restart: unless-stopped + # # https://docs.docker.com/engine/daemon/prometheus/#create-a-prometheus-configuration + # # If you're using Docker Desktop, the --add-host flag is optional. This flag makes sure that the host's internal IP gets exposed to the Prometheus container. + # extra_hosts: + # - "host.docker.internal:host-gateway" + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: all + # capabilities: [gpu] diff --git a/480T/ragflow/entrypoint.sh b/480T/ragflow/entrypoint.sh new file mode 100755 index 0000000..d7ef35b --- /dev/null +++ b/480T/ragflow/entrypoint.sh @@ -0,0 +1,357 @@ +#!/usr/bin/env bash + +set -e + +echo "Start RAGFlow cluster, version: " +cat /ragflow/VERSION + +# ----------------------------------------------------------------------------- +# Usage and command-line argument parsing +# ----------------------------------------------------------------------------- +function usage() { + echo "Usage: $0 [--disable-webserver] [--disable-taskexecutor] [--disable-datasync] [--consumer-no-beg=] [--consumer-no-end=] [--workers=] [--host-id=]" + echo + echo " --disable-webserver Disables the web server (nginx + ragflow_server)." + echo " --disable-taskexecutor Disables task executor workers." + echo " --disable-datasync Disables synchronization of datasource workers." + echo " --enable-mcpserver Enables the MCP server." + echo " --enable-adminserver Enables the Admin server." + echo " --init-model-provider-tables Run model provider table migrations and exit." + echo " --init-superuser Initializes the superuser." + echo " --consumer-no-beg= Start range for consumers (if using range-based)." + echo " --consumer-no-end= End range for consumers (if using range-based)." + echo " --workers= Number of task executors to run (if range is not used)." + echo " --host-id= Unique ID for the host (defaults to \`hostname\`)." + echo + echo "Examples:" + echo " $0 --disable-taskexecutor" + echo " $0 --disable-webserver --consumer-no-beg=0 --consumer-no-end=5" + echo " $0 --disable-webserver --workers=2 --host-id=myhost123" + echo " $0 --enable-mcpserver" + echo " $0 --enable-adminserver" + echo " $0 --init-superuser" + exit 1 +} + +ENABLE_WEBSERVER=1 # Default to enable web server +ENABLE_TASKEXECUTOR=1 # Default to enable task executor +ENABLE_DATASYNC=1 +ENABLE_MCP_SERVER=0 +ENABLE_ADMIN_SERVER=0 # Default close admin server +INIT_SUPERUSER_ARGS="" # Default to not initialize superuser +INIT_MODEL_PROVIDER_TABLES=0 +CONSUMER_NO_BEG=0 +CONSUMER_NO_END=0 +WORKERS=1 + +MCP_HOST="127.0.0.1" +MCP_PORT=9382 +MCP_BASE_URL="http://127.0.0.1:9380" +MCP_SCRIPT_PATH="/ragflow/mcp/server/server.py" +MCP_MODE="self-host" +MCP_HOST_API_KEY="" +MCP_TRANSPORT_SSE_FLAG="--transport-sse-enabled" +MCP_TRANSPORT_STREAMABLE_HTTP_FLAG="--transport-streamable-http-enabled" +MCP_JSON_RESPONSE_FLAG="--json-response" + +# ----------------------------------------------------------------------------- +# Host ID logic: +# 1. By default, use the system hostname if length <= 32 +# 2. Otherwise, use the full MD5 hash of the hostname (32 hex chars) +# ----------------------------------------------------------------------------- +CURRENT_HOSTNAME="$(hostname)" +if [ ${#CURRENT_HOSTNAME} -le 32 ]; then + DEFAULT_HOST_ID="$CURRENT_HOSTNAME" +else + DEFAULT_HOST_ID="$(echo -n "$CURRENT_HOSTNAME" | md5sum | cut -d ' ' -f 1)" +fi + +HOST_ID="$DEFAULT_HOST_ID" + +# Parse arguments +for arg in "$@"; do + case $arg in + --disable-webserver) + ENABLE_WEBSERVER=0 + shift + ;; + --disable-taskexecutor) + ENABLE_TASKEXECUTOR=0 + shift + ;; + --disable-datasync) + ENABLE_DATASYNC=0 + shift + ;; + --enable-mcpserver) + ENABLE_MCP_SERVER=1 + shift + ;; + --enable-adminserver) + ENABLE_ADMIN_SERVER=1 + shift + ;; + --init-model-provider-tables) + INIT_MODEL_PROVIDER_TABLES=1 + shift + ;; + --init-superuser) + INIT_SUPERUSER_ARGS="--init-superuser" + shift + ;; + --mcp-host=*) + MCP_HOST="${arg#*=}" + shift + ;; + --mcp-port=*) + MCP_PORT="${arg#*=}" + shift + ;; + --mcp-base-url=*) + MCP_BASE_URL="${arg#*=}" + shift + ;; + --mcp-mode=*) + MCP_MODE="${arg#*=}" + shift + ;; + --mcp-host-api-key=*) + MCP_HOST_API_KEY="${arg#*=}" + shift + ;; + --mcp-script-path=*) + MCP_SCRIPT_PATH="${arg#*=}" + shift + ;; + --no-transport-sse-enabled) + MCP_TRANSPORT_SSE_FLAG="--no-transport-sse-enabled" + shift + ;; + --no-transport-streamable-http-enabled) + MCP_TRANSPORT_STREAMABLE_HTTP_FLAG="--no-transport-streamable-http-enabled" + shift + ;; + --no-json-response) + MCP_JSON_RESPONSE_FLAG="--no-json-response" + shift + ;; + --consumer-no-beg=*) + CONSUMER_NO_BEG="${arg#*=}" + shift + ;; + --consumer-no-end=*) + CONSUMER_NO_END="${arg#*=}" + shift + ;; + --workers=*) + WORKERS="${arg#*=}" + shift + ;; + --host-id=*) + HOST_ID="${arg#*=}" + shift + ;; + *) + usage + ;; + esac +done + +# ----------------------------------------------------------------------------- +# Replace env variables in the service_conf.yaml file +# ----------------------------------------------------------------------------- +CONF_DIR="/ragflow/conf" +TEMPLATE_FILE="${CONF_DIR}/service_conf.yaml.template" +CONF_FILE="${CONF_DIR}/service_conf.yaml" + +rm -f "${CONF_FILE}" +DEF_ENV_VALUE_PATTERN="\$\{([^:]+):-([^}]+)\}" +while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" =~ DEF_ENV_VALUE_PATTERN ]]; then + varname="${BASH_REMATCH[1]}" + default="${BASH_REMATCH[2]}" + + if [ -n "${!varname}" ]; then + eval "echo \"$line"\" >> "${CONF_FILE}" + else + echo "$line" | sed -E "s/\\\$\{[^:]+:-([^}]+)\}/\1/g" >> "${CONF_FILE}" + fi + else + eval "echo \"$line\"" >> "${CONF_FILE}" + fi +done < "${TEMPLATE_FILE}" + +export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/" +PY=python3 + +# ----------------------------------------------------------------------------- +# Select Nginx Configuration based on API_PROXY_SCHEME +# ----------------------------------------------------------------------------- +NGINX_CONF_DIR="/etc/nginx/conf.d" +if [ -n "$API_PROXY_SCHEME" ]; then + if [[ "${API_PROXY_SCHEME}" == "hybrid" ]]; then + cp -f "$NGINX_CONF_DIR/ragflow.conf.hybrid" "$NGINX_CONF_DIR/ragflow.conf" + echo "Applied nginx config: ragflow.conf.hybrid" + elif [[ "${API_PROXY_SCHEME}" == "go" ]]; then + cp -f "$NGINX_CONF_DIR/ragflow.conf.golang" "$NGINX_CONF_DIR/ragflow.conf" + echo "Applied nginx config: ragflow.conf.golang (default)" + else + cp -f "$NGINX_CONF_DIR/ragflow.conf.python" "$NGINX_CONF_DIR/ragflow.conf" + echo "Applied nginx config: ragflow.conf.python" + fi +else + # Default to python backend + cp -f "$NGINX_CONF_DIR/ragflow.conf.python" "$NGINX_CONF_DIR/ragflow.conf" + echo "Default: applied nginx config: ragflow.conf.python" +fi + +# ----------------------------------------------------------------------------- +# Function(s) +# ----------------------------------------------------------------------------- + +function task_exe() { + local consumer_id="$1" + local host_id="$2" + + JEMALLOC_PATH="$(pkg-config --variable=libdir jemalloc)/libjemalloc.so" + while true; do + LD_PRELOAD="$JEMALLOC_PATH" \ + "$PY" rag/svr/task_executor.py -i "${host_id}_${consumer_id}" -t "common" & + wait; + sleep 1; + done +} + +function start_mcp_server() { + echo "Starting MCP Server on ${MCP_HOST}:${MCP_PORT} with base URL ${MCP_BASE_URL}..." + "$PY" "${MCP_SCRIPT_PATH}" \ + --host="${MCP_HOST}" \ + --port="${MCP_PORT}" \ + --base-url="${MCP_BASE_URL}" \ + --mode="${MCP_MODE}" \ + --api-key="${MCP_HOST_API_KEY}" \ + "${MCP_TRANSPORT_SSE_FLAG}" \ + "${MCP_TRANSPORT_STREAMABLE_HTTP_FLAG}" \ + "${MCP_JSON_RESPONSE_FLAG}" & +} + +function ensure_docling() { + [[ "${USE_DOCLING}" == "true" ]] || { echo "[docling] disabled by USE_DOCLING"; return 0; } + DOCLING_PIN="${DOCLING_VERSION:-==2.71.0}" + "$PY" -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('docling') else 1)" \ + || uv pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --extra-index-url https://pypi.org/simple --no-cache-dir "docling${DOCLING_PIN}" +} + +function ensure_db_init() { + echo "Initializing database tables..." + "$PY" -c "from api.db.db_models import init_database_tables as init_web_db; init_web_db()" + echo "Database tables initialized." +} + +# ----------------------------------------------------------------------------- +# Start components based on flags +# ----------------------------------------------------------------------------- +ensure_docling +ensure_db_init + +if [[ "${INIT_MODEL_PROVIDER_TABLES}" -eq 1 ]]; then + tools/scripts/run_migrations.sh +fi + +if [[ "${ENABLE_ADMIN_SERVER}" -eq 1 ]]; then + if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then + while true; do + echo "Attempt to start Admin python server..." + "$PY" admin/server/admin_server.py + echo "Admin python server started" + sleep 1; + done & + fi + + if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then + while true; do + echo "Starting Admin go server..." + bin/ragflow_server --admin + echo "Admin go server started." + sleep 1; + done & + fi +fi + +if [[ "${ENABLE_WEBSERVER}" -eq 1 ]]; then + echo "Starting nginx..." + /usr/sbin/nginx + + if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then + while true; do + echo "Attempt to start RAGFlow python server..." + "$PY" api/ragflow_server.py ${INIT_SUPERUSER_ARGS} + echo "RAGFlow python server started." + sleep 1; + done & + fi + + if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then + while true; do + echo "Starting RAGFlow go server..." + bin/ragflow_server --api + echo "RAGFlow go server started." + sleep 1; + done & + fi +fi + +if [[ "${ENABLE_DATASYNC}" -eq 1 ]]; then + echo "Starting data sync..." + while true; do + "$PY" rag/svr/sync_data_source.py & + wait; + sleep 1; + done & +fi + +if [[ "${ENABLE_MCP_SERVER}" -eq 1 ]]; then + start_mcp_server +fi + + +if [[ "${ENABLE_TASKEXECUTOR}" -eq 1 ]]; then + if [[ "${CONSUMER_NO_END}" -gt "${CONSUMER_NO_BEG}" ]]; then + if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then + echo "Starting python task executors on host '${HOST_ID}' for IDs in [${CONSUMER_NO_BEG}, ${CONSUMER_NO_END})..." + for (( i=CONSUMER_NO_BEG; i/dev/null; then + echo "Killing process $pid" + kill "$pid" + fi + done + exit 0 +} + +# Trap SIGINT and SIGTERM to invoke cleanup +trap cleanup SIGINT SIGTERM + +# Function to execute task_executor with retry logic +task_exe(){ + local task_id=$1 + local retry_count=0 + while ! $STOP && [ $retry_count -lt $MAX_RETRIES ]; do + echo "Starting task_executor.py for task $task_id (Attempt $((retry_count+1)))" + LD_PRELOAD=$JEMALLOC_PATH $PY rag/svr/task_executor.py -i "$task_id" + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo "task_executor.py for task $task_id exited successfully." + break + else + echo "task_executor.py for task $task_id failed with exit code $EXIT_CODE. Retrying..." >&2 + retry_count=$((retry_count + 1)) + sleep 2 + fi + done + + if [ $retry_count -ge $MAX_RETRIES ]; then + echo "task_executor.py for task $task_id failed after $MAX_RETRIES attempts. Exiting..." >&2 + cleanup + fi +} + +# Function to execute ragflow_server with retry logic +run_server(){ + local server_name="ragflow_server.py" + local -a server_cmd=("$PY" "api/ragflow_server.py") + if [[ "${API_PROXY_SCHEME}" == "go" ]]; then + prepare_for_go + server_name="ragflow_server" + server_cmd=("bin/ragflow_server" "--api") + fi + local retry_count=0 + while ! $STOP && [ $retry_count -lt $MAX_RETRIES ]; do + echo "Starting $server_name (Attempt $((retry_count+1)))" + "${server_cmd[@]}" + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo "$server_name exited successfully." + break + else + echo "$server_name failed with exit code $EXIT_CODE. Retrying..." >&2 + retry_count=$((retry_count + 1)) + sleep 2 + fi + done + + if [ $retry_count -ge $MAX_RETRIES ]; then + echo "$server_name failed after $MAX_RETRIES attempts. Exiting..." >&2 + cleanup + fi +} + +# Function to execute admin_server with retry logic +run_admin_server(){ + local server_name="admin_server.py" + local -a server_cmd=("$PY" "admin/server/admin_server.py") + if [[ "${API_PROXY_SCHEME}" == "go" ]]; then + prepare_for_go + server_name="admin_server" + server_cmd=("bin/ragflow_server" "--admin") + fi + local retry_count=0 + while ! $STOP && [ $retry_count -lt $MAX_RETRIES ]; do + echo "Starting $server_name (Attempt $((retry_count+1)))" + "${server_cmd[@]}" + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo "$server_name exited successfully." + break + else + echo "$server_name failed with exit code $EXIT_CODE. Retrying..." >&2 + retry_count=$((retry_count + 1)) + sleep 2 + fi + done + if [ $retry_count -ge $MAX_RETRIES ]; then + echo "$server_name failed after $MAX_RETRIES attempts. Exiting..." >&2 + cleanup + fi +} + +# Function to execute sync_data_source with retry logic +run_data_sync(){ + local retry_count=0 + while ! $STOP && [ $retry_count -lt $MAX_RETRIES ]; do + echo "Starting sync_data_source.py (Attempt $((retry_count+1)))" + $PY rag/svr/sync_data_source.py + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo "sync_data_source.py exited successfully." + break + else + echo "sync_data_source.py failed with exit code $EXIT_CODE. Retrying..." >&2 + retry_count=$((retry_count + 1)) + sleep 2 + fi + done + + if [ $retry_count -ge $MAX_RETRIES ]; then + echo "sync_data_source.py failed after $MAX_RETRIES attempts. Exiting..." >&2 + cleanup + fi +} + +ensure_db_init() { + echo "Initializing database tables..." + "$PY" -c "from api.db.db_models import init_database_tables as init_web_db; init_web_db()" + echo "Database tables initialized." +} + +run_mysql_migrations() { + tools/scripts/run_migrations.sh +} + +prepare_for_go() { + if [ -d /usr/share/infinity/resource ]; then + echo "Resource directory already exists. Skipping preparation." + return + fi + mkdir -p /usr/share/infinity/resource + if [ "$NEED_MIRROR" == "1" ]; then + git clone --depth 1 --single-branch https://gitee.com/infiniflow/resource /tmp/resource; + else + git clone --depth 1 --single-branch https://github.com/infiniflow/resource.git /tmp/resource; + fi + cp -r /tmp/resource/* /usr/share/infinity/resource + rm -rf /tmp/resource +} + +START_RAGFLOW=0 +START_TASK_EXECUTOR=0 +START_ADMIN=0 +START_DATA_SYNC=0 + +if [ $# -eq 0 ]; then + START_RAGFLOW=1 + START_TASK_EXECUTOR=1 +fi + +for arg in "$@"; do + case $arg in + ragflow|server|webserver) + START_RAGFLOW=1 + ;; + task_executor|task-executor|taskexecutor) + START_TASK_EXECUTOR=1 + ;; + admin|admin_server|admin-server) + START_ADMIN=1 + ;; + data_sync|data-sync|datasync) + START_DATA_SYNC=1 + ;; + all) + START_RAGFLOW=1 + START_TASK_EXECUTOR=1 + START_ADMIN=1 + START_DATA_SYNC=1 + ;; + -h|--help) + usage 0 + ;; + *) + echo "Unknown service type: $arg" >&2 + usage + ;; + esac +done + +if [[ "$START_RAGFLOW" -eq 1 ]]; then + ensure_db_init + run_mysql_migrations +fi + +# Start task executors +if [[ "$START_TASK_EXECUTOR" -eq 1 ]]; then + for ((i=0;i [backup_folder]" + echo "" + echo "OPERATIONS:" + echo " backup - Create backup of all RAGFlow data volumes" + echo " restore - Restore RAGFlow data volumes from backup" + echo " help - Show this help message" + echo "" + echo "OPTIONS:" + echo " -p project_name - Docker Compose project name (default: '$DEFAULT_PROJECT_NAME')" + echo " Use this when you started RAGFlow with 'docker compose -p '" + echo "" + echo "PARAMETERS:" + echo " backup_folder - Name of backup folder (default: '$DEFAULT_BACKUP_FOLDER')" + echo "" + echo "EXAMPLES:" + echo " $0 backup # Backup with default project name 'docker'" + echo " $0 backup my_backup # Backup to './my_backup' folder" + echo " $0 restore # Restore from './backup' folder" + echo " $0 restore my_backup # Restore from './my_backup' folder" + echo " $0 -p ragflow backup # Backup volumes for project 'ragflow'" + echo " $0 -p ragflow restore my_backup # Restore volumes for project 'ragflow'" + echo "" + echo "DOCKER VOLUMES (with default project name '$DEFAULT_PROJECT_NAME'):" + echo " - ${DEFAULT_PROJECT_NAME}_mysql_data (MySQL database)" + echo " - ${DEFAULT_PROJECT_NAME}_minio_data (MinIO object storage)" + echo " - ${DEFAULT_PROJECT_NAME}_redis_data (Redis cache)" + echo " - ${DEFAULT_PROJECT_NAME}_esdata01 (Elasticsearch indices)" + echo "" + echo "NOTE:" + echo " If you started RAGFlow with 'docker compose -p myproject up', the volume" + echo " names will be prefixed with 'myproject' instead of 'docker'. In that case," + echo " use '-p myproject' with this script to match the correct volumes." +} + +# Function to check if Docker is running +check_docker() { + if ! docker info >/dev/null 2>&1; then + echo "❌ Error: Docker is not running or not accessible" + echo "Please start Docker and try again" + exit 1 + fi +} + +# Function to check if volume exists +volume_exists() { + local volume_name=$1 + docker volume inspect "$volume_name" >/dev/null 2>&1 +} + +# Function to check if any containers are using the target volumes +check_containers_using_volumes() { + echo "🔍 Checking for running containers that might be using target volumes..." + + # Get all running containers + local running_containers=$(docker ps --format "{{.Names}}") + + if [ -z "$running_containers" ]; then + echo "✅ No running containers found" + return 0 + fi + + # Check each running container for volume usage + local containers_using_volumes=() + local volume_usage_details=() + + for container in $running_containers; do + # Get container's mount information + local mounts=$(docker inspect "$container" --format '{{range .Mounts}}{{.Source}}{{"|"}}{{end}}' 2>/dev/null || echo "") + + # Check if any of our target volumes are used by this container + for volume in "${VOLUMES[@]}"; do + if echo "$mounts" | grep -q "$volume"; then + containers_using_volumes+=("$container") + volume_usage_details+=("$container -> $volume") + break + fi + done + done + + # If any containers are using our volumes, show error and exit + if [ ${#containers_using_volumes[@]} -gt 0 ]; then + echo "" + echo "❌ ERROR: Found running containers using target volumes!" + echo "" + echo "📋 Running containers status:" + docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" + echo "" + echo "🔗 Volume usage details:" + for detail in "${volume_usage_details[@]}"; do + echo " - $detail" + done + echo "" + if [ "$PROJECT_NAME" = "$DEFAULT_PROJECT_NAME" ]; then + echo "🛑 SOLUTION: Stop the containers before performing backup/restore operations:" + echo " docker compose -f docker/.yml down" + else + echo "🛑 SOLUTION: Stop the containers before performing backup/restore operations:" + echo " docker compose -p $PROJECT_NAME -f docker/.yml down" + fi + echo "" + echo "💡 After backup/restore, you can restart with the corresponding 'up -d' command." + echo "" + exit 1 + fi + + echo "✅ No containers are using target volumes, safe to proceed" + return 0 +} + +# Function to confirm user action +confirm_action() { + local message=$1 + echo -n "$message (y/N): " + read -r response + case "$response" in + [yY]|[yY][eE][sS]) return 0 ;; + *) return 1 ;; + esac +} + +# Function to perform backup +perform_backup() { + local backup_folder=$1 + + echo "🚀 Starting RAGFlow data backup..." + echo "📁 Backup folder: $backup_folder" + echo "🏷️ Project name: $PROJECT_NAME" + echo "" + + # Check if any containers are using the volumes + check_containers_using_volumes + + # Create backup folder if it doesn't exist + mkdir -p "$backup_folder" + + local total=${#VOLUMES[@]} + + # Backup each volume + for i in "${!VOLUMES[@]}"; do + local volume="${VOLUMES[$i]}" + local backup_file="${BACKUP_FILES[$i]}" + local step=$((i + 1)) + + echo "📦 Step $step/$total: Backing up $volume..." + + if volume_exists "$volume"; then + docker run --rm \ + -v "$volume":/source \ + -v "$(pwd)/$backup_folder":/backup \ + alpine tar czf "/backup/$backup_file" -C /source . + echo "✅ Successfully backed up $volume to $backup_folder/$backup_file" + else + echo "⚠️ Warning: Volume $volume does not exist, skipping..." + fi + echo "" + done + + echo "🎉 Backup completed successfully!" + echo "📍 Backup location: $(pwd)/$backup_folder" + + # List backup files with sizes + echo "" + echo "📋 Backup files created:" + for backup_file in "${BACKUP_FILES[@]}"; do + if [ -f "$backup_folder/$backup_file" ]; then + local size=$(ls -lh "$backup_folder/$backup_file" | awk '{print $5}') + echo " - $backup_file ($size)" + fi + done +} + +# Function to perform restore +perform_restore() { + local backup_folder=$1 + + echo "🔄 Starting RAGFlow data restore..." + echo "📁 Backup folder: $backup_folder" + echo "🏷️ Project name: $PROJECT_NAME" + echo "" + + # Check if any containers are using the volumes + check_containers_using_volumes + + # Check if backup folder exists + if [ ! -d "$backup_folder" ]; then + echo "❌ Error: Backup folder '$backup_folder' does not exist" + exit 1 + fi + + # Check if all backup files exist + local missing_files=() + for backup_file in "${BACKUP_FILES[@]}"; do + if [ ! -f "$backup_folder/$backup_file" ]; then + missing_files+=("$backup_file") + fi + done + + if [ ${#missing_files[@]} -gt 0 ]; then + echo "❌ Error: Missing backup files:" + for file in "${missing_files[@]}"; do + echo " - $file" + done + echo "Please ensure all backup files are present in '$backup_folder'" + exit 1 + fi + + # Check for existing volumes and warn user + local existing_volumes=() + for volume in "${VOLUMES[@]}"; do + if volume_exists "$volume"; then + existing_volumes+=("$volume") + fi + done + + if [ ${#existing_volumes[@]} -gt 0 ]; then + echo "⚠️ WARNING: The following Docker volumes already exist:" + for volume in "${existing_volumes[@]}"; do + echo " - $volume" + done + echo "" + echo "🔴 IMPORTANT: Restoring will OVERWRITE existing data!" + echo "💡 Recommendation: Create a backup of your current data first:" + echo " $0 -p $PROJECT_NAME backup current_backup_$(date +%Y%m%d_%H%M%S)" + echo "" + + if ! confirm_action "Do you want to continue with the restore operation?"; then + echo "❌ Restore operation cancelled by user" + exit 0 + fi + fi + + local total=${#VOLUMES[@]} + + # Create volumes and restore data + for i in "${!VOLUMES[@]}"; do + local volume="${VOLUMES[$i]}" + local backup_file="${BACKUP_FILES[$i]}" + local step=$((i + 1)) + + echo "🔧 Step $step/$total: Restoring $volume..." + + # Create volume if it doesn't exist + if ! volume_exists "$volume"; then + echo " 📋 Creating Docker volume: $volume" + docker volume create "$volume" + else + echo " 📋 Using existing Docker volume: $volume" + fi + + # Restore data + echo " 📥 Restoring data from $backup_file..." + docker run --rm \ + -v "$volume":/target \ + -v "$(pwd)/$backup_folder":/backup \ + alpine tar xzf "/backup/$backup_file" -C /target + + echo "✅ Successfully restored $volume" + echo "" + done + + echo "🎉 Restore completed successfully!" + echo "💡 You can now start your RAGFlow services" +} + +# Main script logic +main() { + # Check if Docker is available + check_docker + + # Parse -p flag + PROJECT_NAME="$DEFAULT_PROJECT_NAME" + while [ $# -gt 0 ]; do + case "$1" in + -p) + if [ -z "${2:-}" ]; then + echo "❌ Error: -p requires a project name argument" + exit 1 + fi + PROJECT_NAME="$2" + shift 2 + ;; + *) + break + ;; + esac + done + + # Build volume names based on project name + build_volume_names + + # Parse remaining positional arguments + local operation=${1:-} + local backup_folder=${2:-$DEFAULT_BACKUP_FOLDER} + + # Handle help or no arguments + if [ -z "$operation" ] || [ "$operation" = "help" ] || [ "$operation" = "-h" ] || [ "$operation" = "--help" ]; then + show_help + exit 0 + fi + + # Validate operation + case "$operation" in + backup) + perform_backup "$backup_folder" + ;; + restore) + perform_restore "$backup_folder" + ;; + *) + echo "❌ Error: Invalid operation '$operation'" + echo "" + show_help + exit 1 + ;; + esac +} + +# Run main function with all arguments +main "$@" diff --git a/480T/ragflow/nginx/nginx.conf b/480T/ragflow/nginx/nginx.conf new file mode 100755 index 0000000..41b9915 --- /dev/null +++ b/480T/ragflow/nginx/nginx.conf @@ -0,0 +1,33 @@ +user root; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + client_max_body_size 1024M; + + include /etc/nginx/conf.d/ragflow.conf; +} + diff --git a/480T/ragflow/nginx/proxy.conf b/480T/ragflow/nginx/proxy.conf new file mode 100755 index 0000000..96bcb2f --- /dev/null +++ b/480T/ragflow/nginx/proxy.conf @@ -0,0 +1,12 @@ +proxy_set_header Host $host; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_http_version 1.1; +proxy_set_header Connection ""; +proxy_buffering off; +proxy_read_timeout 3600s; +proxy_send_timeout 3600s; +proxy_buffer_size 1024k; +proxy_buffers 16 1024k; +proxy_busy_buffers_size 2048k; +proxy_temp_file_write_size 2048k; \ No newline at end of file diff --git a/480T/ragflow/nginx/ragflow.conf.golang b/480T/ragflow/nginx/ragflow.conf.golang new file mode 100755 index 0000000..af6607a --- /dev/null +++ b/480T/ragflow/nginx/ragflow.conf.golang @@ -0,0 +1,38 @@ +server { + listen 80; + server_name _; + root /ragflow/web/dist; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 9; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + location ~ ^/api/v1/admin { + proxy_pass http://127.0.0.1:9383; + include proxy.conf; + } + + location ~ ^/api/v1/skills { + proxy_pass http://127.0.0.1:9384; + include proxy.conf; + } + + location ~ ^/(v1|api) { + proxy_pass http://127.0.0.1:9384; + include proxy.conf; + } + + location / { + index index.html; + try_files $uri $uri/ /index.html; + } + + # Cache-Control: max-age Expires + location ~ ^/static/(css|js|media)/ { + expires 10y; + access_log off; + } +} diff --git a/480T/ragflow/nginx/ragflow.conf.hybrid b/480T/ragflow/nginx/ragflow.conf.hybrid new file mode 100755 index 0000000..3745c78 --- /dev/null +++ b/480T/ragflow/nginx/ragflow.conf.hybrid @@ -0,0 +1,83 @@ +server { + listen 80; + server_name _; + root /ragflow/web/dist; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 9; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + location ~ ^/api/v1/admin/roles_with_permission { + proxy_pass http://127.0.0.1:9381; + include proxy.conf; + } + + location ~ ^/api/v1/admin/sandbox { + proxy_pass http://127.0.0.1:9381; + include proxy.conf; + } + + location ~ ^/api/v1/admin/roles { + proxy_pass http://127.0.0.1:9381; + include proxy.conf; + } + + location ~ ^/api/v1/admin/whitelist { + proxy_pass http://127.0.0.1:9381; + include proxy.conf; + } + + location ~ ^/api/v1/admin/variables { + proxy_pass http://127.0.0.1:9381; + include proxy.conf; + } + + location ~ ^/api/v1/admin { + proxy_pass http://127.0.0.1:9383; + include proxy.conf; + } + + location ~ ^/api/v1/skills { + proxy_pass http://127.0.0.1:9384; + include proxy.conf; + } + + location ~ ^/api/v1/datasets/search { + proxy_pass http://127.0.0.1:9384; + include proxy.conf; + } + + location ~ ^/api/v1/chat/completions { + proxy_pass http://127.0.0.1:9384; + include proxy.conf; + } + + location ~ ^/v1/system/config { + proxy_pass http://127.0.0.1:9384; + include proxy.conf; + } + + location ~ ^/v1/user/(login|logout) { + proxy_pass http://127.0.0.1:9384; + include proxy.conf; + } + + location ~ ^/(v1|api) { + proxy_pass http://127.0.0.1:9380; + include proxy.conf; + } + + location / { + index index.html; + try_files $uri $uri/ /index.html; + } + + # Cache-Control: max-age Expires + location ~ ^/static/(css|js|media)/ { + expires 10y; + access_log off; + } +} \ No newline at end of file diff --git a/480T/ragflow/nginx/ragflow.conf.python b/480T/ragflow/nginx/ragflow.conf.python new file mode 100755 index 0000000..d000e41 --- /dev/null +++ b/480T/ragflow/nginx/ragflow.conf.python @@ -0,0 +1,34 @@ +server { + listen 80; + server_name _; + root /ragflow/web/dist; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 9; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + location ~ ^/api/v1/admin { + proxy_pass http://localhost:9381; + include proxy.conf; + } + + location ~ ^/(v1|api) { + proxy_pass http://localhost:9380; + include proxy.conf; + } + + + location / { + index index.html; + try_files $uri $uri/ /index.html; + } + + # Cache-Control: max-age Expires + location ~ ^/static/(css|js|media)/ { + expires 10y; + access_log off; + } +} diff --git a/480T/ragflow/nginx/ragflow.https.conf b/480T/ragflow/nginx/ragflow.https.conf new file mode 100755 index 0000000..2ef4cd0 --- /dev/null +++ b/480T/ragflow/nginx/ragflow.https.conf @@ -0,0 +1,46 @@ +server { + listen 80; + server_name your-ragflow-domain.com; + return 301 https://$host$request_uri; +} + + + +server { + listen 443 ssl; + server_name your-ragflow-domain.com; + + ssl_certificate /etc/nginx/ssl/fullchain.pem; + ssl_certificate_key /etc/nginx/ssl/privkey.pem; + + root /ragflow/web/dist; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 9; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + location ~ ^/api/v1/admin { + proxy_pass http://localhost:9381; + include proxy.conf; + } + + location ~ ^/(v1|api) { + proxy_pass http://localhost:9380; + include proxy.conf; + } + + + location / { + index index.html; + try_files $uri $uri/ /index.html; + } + + # Cache-Control: max-age~@~AExpires + location ~ ^/static/(css|js|media)/ { + expires 10y; + access_log off; + } +} diff --git a/480T/ragflow/oceanbase-entrypoint.sh b/480T/ragflow/oceanbase-entrypoint.sh new file mode 100755 index 0000000..2124e0d --- /dev/null +++ b/480T/ragflow/oceanbase-entrypoint.sh @@ -0,0 +1,115 @@ +#!/bin/bash +set -euo pipefail + +log() { + echo "[ragflow-ob-entrypoint] $*" +} + +ob_sys() { + obclient -h127.0.0.1 -P2881 -uroot@sys -p"${OB_SYS_PASSWORD}" -Doceanbase -A -N -e "$1" +} + +ob_tenant_with_password() { + obclient -h127.0.0.1 -P2881 -uroot@"${OB_TENANT_NAME}" -p"${OB_TENANT_PASSWORD}" -A -N -e "$1" +} + +ob_tenant_without_password() { + obclient -h127.0.0.1 -P2881 -uroot@"${OB_TENANT_NAME}" -A -N -e "$1" +} + +wait_for_sys() { + for _ in $(seq 1 120); do + if ob_sys "SELECT 1" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +tenant_exists() { + ob_sys "SELECT 1 FROM oceanbase.DBA_OB_TENANTS WHERE tenant_name='${OB_TENANT_NAME}'" 2>/dev/null | grep -q '^1$' +} + +wait_for_tenant_password() { + for _ in $(seq 1 60); do + if ob_tenant_with_password "SELECT 1" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +ensure_tenant_ready() { + if ob_tenant_with_password "SELECT 1" >/dev/null 2>&1; then + log "tenant ${OB_TENANT_NAME} accepts the configured password" + return 0 + fi + + if tenant_exists; then + for _ in $(seq 1 120); do + if ob_tenant_with_password "SELECT 1" >/dev/null 2>&1; then + log "tenant ${OB_TENANT_NAME} became reachable with the configured password" + return 0 + fi + + if ob_tenant_without_password "SELECT 1" >/dev/null 2>&1; then + log "tenant ${OB_TENANT_NAME} exists with an empty root password; applying OB_TENANT_PASSWORD" + ob_tenant_without_password "ALTER USER root IDENTIFIED BY '${OB_TENANT_PASSWORD}'" >/dev/null 2>&1 || { + log "warning: failed to update the tenant root password" + return 1 + } + wait_for_tenant_password && return 0 + log "warning: tenant password update did not become effective in time" + return 1 + fi + + sleep 1 + done + + log "warning: tenant ${OB_TENANT_NAME} exists but never accepted the configured or empty password during reconciliation" + return 1 + fi + + log "tenant ${OB_TENANT_NAME} is missing; creating it with the configured password" + if ! obd cluster tenant create obcluster -n "${OB_TENANT_NAME}" -o "${OB_SCENARIO:-htap}" --password "${OB_TENANT_PASSWORD}" >/dev/null 2>&1; then + log "warning: failed to create tenant ${OB_TENANT_NAME}" + return 1 + fi + + if wait_for_tenant_password; then + return 0 + fi + + log "warning: tenant ${OB_TENANT_NAME} did not become connectable in time" + return 1 +} + +ensure_database() { + if ob_tenant_with_password "CREATE DATABASE IF NOT EXISTS \`${OCEANBASE_DOC_DBNAME}\`" >/dev/null 2>&1; then + log "database ${OCEANBASE_DOC_DBNAME} is ready in tenant ${OB_TENANT_NAME}" + return 0 + fi + + log "warning: failed to ensure database ${OCEANBASE_DOC_DBNAME}" + return 1 +} + +reconcile_oceanbase() { + if ! wait_for_sys; then + log "warning: sys tenant never became reachable; skipping reconciliation" + return 0 + fi + + ensure_tenant_ready || return 0 + ensure_database || return 0 +} + +/usr/sbin/sshd +/root/boot/start.sh & +start_pid=$! + +reconcile_oceanbase || true & + +wait "${start_pid}" diff --git a/480T/ragflow/oceanbase/init.d/vec_memory.sql b/480T/ragflow/oceanbase/init.d/vec_memory.sql new file mode 100755 index 0000000..f4c283f --- /dev/null +++ b/480T/ragflow/oceanbase/init.d/vec_memory.sql @@ -0,0 +1 @@ +ALTER SYSTEM SET ob_vector_memory_limit_percentage = 30; \ No newline at end of file diff --git a/480T/ragflow/service_conf.yaml.template b/480T/ragflow/service_conf.yaml.template new file mode 100755 index 0000000..25c0ad6 --- /dev/null +++ b/480T/ragflow/service_conf.yaml.template @@ -0,0 +1,209 @@ +ragflow: + host: ${RAGFLOW_HOST:-0.0.0.0} + http_port: 9380 +admin: + host: ${RAGFLOW_HOST:-0.0.0.0} + http_port: 9381 +mysql: + name: '${MYSQL_DBNAME:-rag_flow}' + user: '${MYSQL_USER:-root}' + password: '${MYSQL_PASSWORD:-infini_rag_flow}' + host: '${MYSQL_HOST:-mysql}' + port: ${MYSQL_PORT:-3306} + max_connections: 900 + stale_timeout: 300 + max_allowed_packet: ${MYSQL_MAX_PACKET:-1073741824} +minio: + user: '${MINIO_USER:-rag_flow}' + password: '${MINIO_PASSWORD:-infini_rag_flow}' + host: '${MINIO_HOST:-minio}:9000' + bucket: '${MINIO_BUCKET:-}' + prefix_path: '${MINIO_PREFIX_PATH:-}' + # optional: set to true for HTTPS (SSL/TLS). Used by MinIO client and health check. + # secure: ${MINIO_SECURE:-false} + # optional: set to false to allow self-signed certificates (e.g. in development). + # verify: ${MINIO_VERIFY:-true} +es: + hosts: 'http://${ES_HOST:-es01}:9200' + username: '${ES_USER:-elastic}' + password: '${ELASTIC_PASSWORD:-infini_rag_flow}' +os: + hosts: 'http://${OS_HOST:-opensearch01}:9201' + username: '${OS_USER:-admin}' + password: '${OPENSEARCH_PASSWORD:-infini_rag_flow_OS_01}' + # Optional hybrid (BM25 + KNN) search tuning. The connector self-provisions the + # normalization search pipeline on start-up (requires OpenSearch >= 2.10). + # hybrid_search_pipeline: 'ragflow_hybrid_pipeline' + # hybrid_search_weights: [0.5, 0.5] # [text/BM25 leg, vector/KNN leg] +infinity: + uri: '${INFINITY_HOST:-infinity}:23817' + postgres_port: 5432 + db_name: 'default_db' +oceanbase: + scheme: 'oceanbase' # set 'mysql' to create connection using mysql config + config: + db_name: '${OCEANBASE_DOC_DBNAME:-test}' + user: '${OCEANBASE_USER:-root@ragflow}' + password: '${OCEANBASE_PASSWORD:-infini_rag_flow}' + host: '${OCEANBASE_HOST:-oceanbase}' + port: ${OCEANBASE_PORT:-2881} +seekdb: + scheme: 'oceanbase' # SeekDB is the lite version of OceanBase + config: + db_name: '${SEEKDB_DOC_DBNAME:-ragflow_doc}' + user: '${SEEKDB_USER:-root}' + password: '${SEEKDB_PASSWORD:-infini_rag_flow}' + host: '${SEEKDB_HOST:-seekdb}' + port: ${SEEKDB_PORT:-2881} +redis: + db: 1 + username: '${REDIS_USERNAME:-}' + password: '${REDIS_PASSWORD:-infini_rag_flow}' + host: '${REDIS_HOST:-redis}:6379' +nats: + host: ${NATS_HOST:-0.0.0.0} + port: ${NATS_PORT:-4222} +clickhouse: + host: '${CLICKHOUSE_HOST:-clickhouse}' + port: ${CLICKHOUSE_TCP_PORT:-9000} + user: '${CLICKHOUSE_USER:-ragflow}' + password: '${CLICKHOUSE_PASSWORD:-infini_rag_flow}' + database: '${CLICKHOUSE_DATABASE:-ragflow}' +otel: + host: ${OTEL_HOST:-0.0.0.0} + port: ${OTEL_PORT:-4318} + secure: false + sample_ratio: 1.0 + stdout: false + enable: false +ingestor: + mq_type: 'nats' +file_syncer: + max_concurrent_syncs: 4 + sync_interval: 3 +user_default_llm: + default_models: + embedding_model: + api_key: 'xxx' + base_url: 'http://${TEI_HOST}:80' +# postgres: +# name: '${POSTGRES_DBNAME:-rag_flow}' +# user: '${POSTGRES_USER:-rag_flow}' +# password: '${POSTGRES_PASSWORD:-infini_rag_flow}' +# host: '${POSTGRES_HOST:-postgres}' +# port: 5432 +# max_connections: 100 +# stale_timeout: 30 +# s3: +# access_key: 'access_key' +# secret_key: 'secret_key' +# region: 'region' +# endpoint_url: 'endpoint_url' +# bucket: 'bucket' +# prefix_path: 'prefix_path' +# signature_version: 'v4' +# addressing_style: 'path' +# oss: +# access_key: '${ACCESS_KEY}' +# secret_key: '${SECRET_KEY}' +# endpoint_url: '${ENDPOINT}' +# region: '${REGION}' +# bucket: '${BUCKET}' +# prefix_path: '${OSS_PREFIX_PATH}' +# signature_version: 's3' +# addressing_style: 'virtual' +# azure: +# auth_type: 'sas' +# container_url: 'container_url' +# sas_token: 'sas_token' +# azure: +# auth_type: 'spn' +# account_url: 'account_url' +# client_id: 'client_id' +# secret: 'secret' +# tenant_id: 'tenant_id' +# container_name: 'container_name' +# cloud: 'public' # Azure cloud: 'public', 'china', 'government', or 'germany' +# gcs: +# bucket: '${GCS_BUCKET:-ragflow}' +# prefix_path: '${GCS_PREFIX_PATH:-}' +# endpoint_url: '${GCS_ENDPOINT:-}' +# The OSS object storage uses the MySQL configuration above by default. If you need to switch to another object storage service, please uncomment and configure the following parameters. +# opendal: +# scheme: 'mysql' # Storage type, such as s3, oss, azure, etc. +# config: +# oss_table: 'opendal_storage' +# user_default_llm: +# factory: 'BAAI' +# api_key: 'backup' +# base_url: 'backup_base_url' +# default_models: +# chat_model: +# name: 'qwen2.5-7b-instruct' +# factory: 'xxxx' +# api_key: 'xxxx' +# base_url: 'https://api.xx.com' +# embedding_model: +# name: 'bge-m3' +# rerank_model: 'bge-reranker-v2' +# asr_model: +# model: 'whisper-large-v3' # alias of name +# image2text_model: '' +# oauth: +# oauth2: +# display_name: "OAuth2" +# client_id: "your_client_id" +# client_secret: "your_client_secret" +# authorization_url: "https://your-oauth-provider.com/oauth/authorize" +# token_url: "https://your-oauth-provider.com/oauth/token" +# userinfo_url: "https://your-oauth-provider.com/oauth/userinfo" +# redirect_uri: "https://your-app.com/v1/user/oauth/callback/oauth2" +# oidc: +# display_name: "OIDC" +# client_id: "your_client_id" +# client_secret: "your_client_secret" +# issuer: "https://your-oauth-provider.com/oidc" +# scope: "openid email profile" +# redirect_uri: "https://your-app.com/v1/user/oauth/callback/oidc" +# github: +# type: "github" +# icon: "github" +# display_name: "Github" +# client_id: "your_client_id" +# client_secret: "your_client_secret" +# redirect_uri: "https://your-app.com/v1/user/oauth/callback/github" +# authentication: +# client: +# switch: false +# http_app_key: +# http_secret_key: +# site: +# switch: false +# disable_password_login: false +# permission: +# switch: false +# component: false +# dataset: false +# smtp: +# mail_server: "" +# mail_port: 465 +# mail_use_ssl: true +# mail_use_tls: false +# mail_username: "" +# mail_password: "" +# mail_default_sender: +# - "RAGFlow" # display name +# - "" # sender email address +# mail_frontend_url: "https://your-frontend.example.com" +# tcadp_config: +# secret_id: '${TENCENT_SECRET_ID}' +# secret_key: '${TENCENT_SECRET_KEY}' +# region: '${TENCENT_REGION}' +# log: +# level: info # debug | info | warn | error +# format: text # text | json (reserved; text only today) +# path: server_main.log # empty/missing = binary default (server_main.log / admin_server.log / ingestion_server.log) +# max_size: 100 # MB before rotation +# max_backups: 10 # retained rotated files +# max_age: 30 # days +# compress: true # gzip rotated files; default true. Set false to disable.