final version
This commit is contained in:
@@ -4,15 +4,19 @@ from typing import Optional, Tuple
|
||||
import io
|
||||
import os
|
||||
import logging
|
||||
import shutil
|
||||
|
||||
from app.config.settings import settings
|
||||
|
||||
|
||||
class MinioClient:
|
||||
"""MinIO客户端类"""
|
||||
"""MinIO客户端类,支持本地存储作为备选"""
|
||||
|
||||
def __init__(self):
|
||||
"""初始化MinIO客户端"""
|
||||
self.local_storage_path = "data_storage/local_uploads"
|
||||
os.makedirs(self.local_storage_path, exist_ok=True)
|
||||
|
||||
try:
|
||||
self.client = Minio(
|
||||
settings.MINIO_ENDPOINT,
|
||||
@@ -21,16 +25,24 @@ class MinioClient:
|
||||
secure=settings.MINIO_SECURE
|
||||
)
|
||||
self.bucket_name = settings.MINIO_BUCKET_NAME
|
||||
self.is_connected = True # 先设置为True,这样在调用其他方法时不会报错
|
||||
|
||||
# 测试真实连接
|
||||
self.client.list_buckets()
|
||||
self.is_connected = True
|
||||
logging.info("MinIO connected successfully")
|
||||
|
||||
# 确保存储桶存在
|
||||
self._ensure_bucket_exists()
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to connect to MinIO: {e}. Running in offline mode.")
|
||||
logging.warning(f"Failed to connect to MinIO: {e}. Using local storage.")
|
||||
self.client = None
|
||||
self.bucket_name = settings.MINIO_BUCKET_NAME
|
||||
self.is_connected = False
|
||||
|
||||
def _get_local_path(self, object_name: str) -> str:
|
||||
"""获取本地存储路径"""
|
||||
return os.path.join(self.local_storage_path, object_name)
|
||||
|
||||
def _ensure_bucket_exists(self):
|
||||
"""确保存储桶存在"""
|
||||
if not self.is_connected:
|
||||
@@ -60,24 +72,32 @@ class MinioClient:
|
||||
return False
|
||||
|
||||
def upload_from_bytes(self, data: bytes, object_name: str) -> bool:
|
||||
"""从字节数据上传文件"""
|
||||
if not self.is_connected:
|
||||
logging.warning("MinIO is not connected. Upload skipped.")
|
||||
return False
|
||||
"""从字节数据上传文件,优先使用MinIO,失败则使用本地存储"""
|
||||
if self.is_connected:
|
||||
try:
|
||||
import io
|
||||
file_obj = io.BytesIO(data)
|
||||
self.client.put_object(
|
||||
self.bucket_name,
|
||||
object_name,
|
||||
file_obj,
|
||||
length=len(data),
|
||||
part_size=10*1024*1024
|
||||
)
|
||||
return True
|
||||
except S3Error as e:
|
||||
logging.warning(f"MinIO upload error: {e}, falling back to local storage")
|
||||
|
||||
# 使用本地存储作为备选
|
||||
try:
|
||||
import io
|
||||
file_obj = io.BytesIO(data)
|
||||
self.client.put_object(
|
||||
self.bucket_name,
|
||||
object_name,
|
||||
file_obj,
|
||||
length=len(data),
|
||||
part_size=10*1024*1024
|
||||
)
|
||||
local_path = self._get_local_path(object_name)
|
||||
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
||||
with open(local_path, 'wb') as f:
|
||||
f.write(data)
|
||||
logging.info(f"File saved to local storage: {local_path}")
|
||||
return True
|
||||
except S3Error as e:
|
||||
logging.warning(f"MinIO upload error: {e}")
|
||||
except Exception as e:
|
||||
logging.error(f"Local storage save error: {e}")
|
||||
return False
|
||||
|
||||
def upload_fileobj(self, file_obj: io.BytesIO, object_name: str, content_type: str = "application/octet-stream") -> bool:
|
||||
@@ -118,38 +138,54 @@ class MinioClient:
|
||||
return False
|
||||
|
||||
def get_object(self, object_name: str) -> Optional[bytes]:
|
||||
"""获取对象内容"""
|
||||
if not self.is_connected:
|
||||
logging.warning("MinIO is not connected. Get object skipped.")
|
||||
return None
|
||||
"""获取对象内容,优先使用MinIO,失败则使用本地存储"""
|
||||
if self.is_connected:
|
||||
try:
|
||||
response = self.client.get_object(
|
||||
self.bucket_name,
|
||||
object_name
|
||||
)
|
||||
data = response.read()
|
||||
response.close()
|
||||
response.release_conn()
|
||||
return data
|
||||
except S3Error as e:
|
||||
logging.warning(f"MinIO get object error: {e}, falling back to local storage")
|
||||
|
||||
# 使用本地存储作为备选
|
||||
try:
|
||||
response = self.client.get_object(
|
||||
self.bucket_name,
|
||||
object_name
|
||||
)
|
||||
data = response.read()
|
||||
response.close()
|
||||
response.release_conn()
|
||||
return data
|
||||
except S3Error as e:
|
||||
logging.warning(f"MinIO get object error: {e}")
|
||||
local_path = self._get_local_path(object_name)
|
||||
if os.path.exists(local_path):
|
||||
with open(local_path, 'rb') as f:
|
||||
return f.read()
|
||||
else:
|
||||
logging.warning(f"File not found in local storage: {local_path}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logging.error(f"Local storage get error: {e}")
|
||||
return None
|
||||
|
||||
def delete_object(self, object_name: str) -> bool:
|
||||
"""删除对象"""
|
||||
if not self.is_connected:
|
||||
logging.warning("MinIO is not connected. Delete object skipped.")
|
||||
return False
|
||||
"""删除对象,优先使用MinIO,失败则使用本地存储"""
|
||||
if self.is_connected:
|
||||
try:
|
||||
self.client.remove_object(
|
||||
self.bucket_name,
|
||||
object_name
|
||||
)
|
||||
return True
|
||||
except S3Error as e:
|
||||
logging.warning(f"MinIO delete error: {e}")
|
||||
|
||||
# 使用本地存储作为备选
|
||||
try:
|
||||
self.client.remove_object(
|
||||
self.bucket_name,
|
||||
object_name
|
||||
)
|
||||
return True
|
||||
except S3Error as e:
|
||||
logging.warning(f"MinIO delete error: {e}")
|
||||
local_path = self._get_local_path(object_name)
|
||||
if os.path.exists(local_path):
|
||||
os.remove(local_path)
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
logging.error(f"Local storage delete error: {e}")
|
||||
return False
|
||||
|
||||
def list_objects(self, prefix: str = "") -> list:
|
||||
|
||||
Reference in New Issue
Block a user