feat: persist photos and enrich conversation history
This commit is contained in:
@@ -6,9 +6,6 @@ from pathlib import Path
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from docx import Document as DocxDocument
|
||||
from openai import AsyncOpenAI
|
||||
from pypdf import PdfReader
|
||||
@@ -17,38 +14,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
import settings
|
||||
from db.models import KnowledgeBase, KnowledgeChunk, KnowledgeDocument, ModelResource
|
||||
|
||||
|
||||
def _s3_client():
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.S3_ENDPOINT_URL,
|
||||
aws_access_key_id=settings.S3_ACCESS_KEY,
|
||||
aws_secret_access_key=settings.S3_SECRET_KEY,
|
||||
region_name=settings.S3_REGION,
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
)
|
||||
|
||||
|
||||
def _ensure_bucket_and_put(key: str, data: bytes, mime_type: str) -> None:
|
||||
client = _s3_client()
|
||||
try:
|
||||
client.head_bucket(Bucket=settings.S3_BUCKET)
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status != 404:
|
||||
raise
|
||||
client.create_bucket(Bucket=settings.S3_BUCKET)
|
||||
client.put_object(Bucket=settings.S3_BUCKET, Key=key, Body=data, ContentType=mime_type)
|
||||
|
||||
|
||||
def _delete_object(key: str) -> None:
|
||||
_s3_client().delete_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
|
||||
|
||||
def _get_object(key: str) -> bytes:
|
||||
response = _s3_client().get_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
return response["Body"].read()
|
||||
from services.object_storage import delete_object, get_object, put_object
|
||||
|
||||
|
||||
def extract_text(filename: str, data: bytes) -> str:
|
||||
@@ -122,7 +88,7 @@ async def create_document(
|
||||
safe_name = Path(name).name
|
||||
extension = ".txt" if source_type == "text" else Path(safe_name).suffix
|
||||
storage_key = f"knowledge/{kb.id}/{document_id}/source{extension}"
|
||||
await asyncio.to_thread(_ensure_bucket_and_put, storage_key, raw_data, mime_type)
|
||||
await asyncio.to_thread(put_object, storage_key, raw_data, mime_type)
|
||||
|
||||
document = KnowledgeDocument(
|
||||
id=document_id,
|
||||
@@ -158,7 +124,7 @@ async def process_document(document_id: str) -> None:
|
||||
await session.commit()
|
||||
|
||||
try:
|
||||
data = await asyncio.to_thread(_get_object, document.storage_key)
|
||||
data = await asyncio.to_thread(get_object, document.storage_key)
|
||||
text = (
|
||||
data.decode("utf-8", errors="replace")
|
||||
if document.source_type == "text"
|
||||
@@ -245,4 +211,4 @@ async def search(
|
||||
|
||||
async def delete_storage_object(document: KnowledgeDocument) -> None:
|
||||
if document.storage_key:
|
||||
await asyncio.to_thread(_delete_object, document.storage_key)
|
||||
await asyncio.to_thread(delete_object, document.storage_key)
|
||||
|
||||
Reference in New Issue
Block a user