update: uploads
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
API_KEY=
|
API_KEY=
|
||||||
MODEL_NAME=gpt-4.1
|
MODEL_NAME=gpt-4.1
|
||||||
DMXAPI_URL=https://www.dmxapi.cn/v1/responses
|
OPENAI_API_URL=https://api.openai.com/v1/responses
|
||||||
JWT_SECRET_KEY=please_change_me
|
JWT_SECRET_KEY=please_change_me
|
||||||
JWT_ALGORITHM=HS256
|
JWT_ALGORITHM=HS256
|
||||||
ACCESS_TOKEN_EXPIRE_MINUTES=720
|
ACCESS_TOKEN_EXPIRE_MINUTES=720
|
||||||
|
|||||||
11
README.md
11
README.md
@@ -30,6 +30,14 @@
|
|||||||
3. 打开:
|
3. 打开:
|
||||||
- `http://127.0.0.1:5173`
|
- `http://127.0.0.1:5173`
|
||||||
|
|
||||||
|
## 导入中心(队列)说明
|
||||||
|
|
||||||
|
- 导入任务使用**持久化队列**:任务与文件项写入数据库(`import_jobs` / `import_job_items`),后端单消费者按创建时间 FIFO 串行执行。
|
||||||
|
- 状态枚举:`queued`(排队中)、`running`(执行中)、`success`、`failed`、`cancelled`、`retrying`。
|
||||||
|
- 刷新页面后,前端会请求 `GET /api/import/jobs?status=queued,running` 恢复未完成任务并轮询进度;无需依赖本地缓存。
|
||||||
|
- 失败项可通过「重试失败项」再次入队(新建任务);排队中/执行中任务可取消。
|
||||||
|
- 上传文件按任务与序号存为唯一路径(`upload_dir/{job_id}/{seq}_{filename}`),避免同名覆盖。
|
||||||
|
|
||||||
## 默认登录
|
## 默认登录
|
||||||
|
|
||||||
- 用户名:`admin`
|
- 用户名:`admin`
|
||||||
@@ -38,8 +46,9 @@
|
|||||||
## 功能清单
|
## 功能清单
|
||||||
|
|
||||||
- 题目 CRUD、搜索、筛选、批量删除、批量更新
|
- 题目 CRUD、搜索、筛选、批量删除、批量更新
|
||||||
- AI 智能导入(PDF/Word -> DMXAPI -> 预览 -> 确认保存)
|
- AI 智能导入(PDF/Word -> OpenAI兼容接口 -> 预览 -> 确认保存)
|
||||||
- Excel 批量导入、模板下载、导出 JSON/CSV/Excel
|
- Excel 批量导入、模板下载、导出 JSON/CSV/Excel
|
||||||
|
- **导入中心(持久化队列)**:严格 FIFO 串行执行,任务状态持久化到数据库;刷新页面后自动恢复未完成任务列表;支持取消排队中/执行中任务、对失败项一键重试入队。
|
||||||
- 分类树管理(章节/知识点)
|
- 分类树管理(章节/知识点)
|
||||||
- 练习模式(抽题、判题、解析反馈)
|
- 练习模式(抽题、判题、解析反馈)
|
||||||
- 仪表盘统计(总量、题型、难度、章节、导入历史)
|
- 仪表盘统计(总量、题型、难度、章节、导入历史)
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
|
|||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
api_key: str = ""
|
api_key: str = ""
|
||||||
model_name: str = "gpt-4.1"
|
model_name: str = "gpt-4.1"
|
||||||
dmxapi_url: str = "https://www.dmxapi.cn/v1/responses"
|
openai_api_url: str = "https://api.openai.com/v1/responses"
|
||||||
jwt_secret_key: str = "change-me-in-env"
|
jwt_secret_key: str = "change-me-in-env"
|
||||||
jwt_algorithm: str = "HS256"
|
jwt_algorithm: str = "HS256"
|
||||||
access_token_expire_minutes: int = 60 * 12
|
access_token_expire_minutes: int = 60 * 12
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
import threading
|
||||||
|
|
||||||
from backend.database import Base, engine
|
from backend.database import Base, engine, SessionLocal
|
||||||
from backend.routers import auth, categories, exports, imports, practice, questions, stats
|
from backend.routers import auth, categories, exports, imports, practice, questions, stats
|
||||||
|
from backend.services.import_queue_service import reset_stale_running_jobs, run_worker_loop
|
||||||
|
|
||||||
app = FastAPI(title="Problem Bank API", version="1.0.0")
|
app = FastAPI(title="Problem Bank API", version="1.0.0")
|
||||||
|
|
||||||
@@ -16,6 +18,17 @@ app.add_middleware(
|
|||||||
|
|
||||||
Base.metadata.create_all(bind=engine)
|
Base.metadata.create_all(bind=engine)
|
||||||
|
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
def startup() -> None:
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
reset_stale_running_jobs(db)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
thread = threading.Thread(target=run_worker_loop, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
app.include_router(auth.router, prefix="/api/auth", tags=["auth"])
|
app.include_router(auth.router, prefix="/api/auth", tags=["auth"])
|
||||||
app.include_router(questions.router, prefix="/api/questions", tags=["questions"])
|
app.include_router(questions.router, prefix="/api/questions", tags=["questions"])
|
||||||
app.include_router(imports.router, prefix="/api/import", tags=["imports"])
|
app.include_router(imports.router, prefix="/api/import", tags=["imports"])
|
||||||
|
|||||||
@@ -1,11 +1,29 @@
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import DateTime, ForeignKey, Integer, Text
|
from sqlalchemy import DateTime, ForeignKey, Integer, String, Text
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from backend.database import Base
|
from backend.database import Base
|
||||||
|
|
||||||
|
|
||||||
|
# 导入任务/文件项统一状态
|
||||||
|
JOB_STATUS_QUEUED = "queued"
|
||||||
|
JOB_STATUS_RUNNING = "running"
|
||||||
|
JOB_STATUS_SUCCESS = "success"
|
||||||
|
JOB_STATUS_FAILED = "failed"
|
||||||
|
JOB_STATUS_CANCELLED = "cancelled"
|
||||||
|
JOB_STATUS_RETRYING = "retrying"
|
||||||
|
|
||||||
|
JOB_STATUSES = (
|
||||||
|
JOB_STATUS_QUEUED,
|
||||||
|
JOB_STATUS_RUNNING,
|
||||||
|
JOB_STATUS_SUCCESS,
|
||||||
|
JOB_STATUS_FAILED,
|
||||||
|
JOB_STATUS_CANCELLED,
|
||||||
|
JOB_STATUS_RETRYING,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Question(Base):
|
class Question(Base):
|
||||||
__tablename__ = "questions"
|
__tablename__ = "questions"
|
||||||
|
|
||||||
@@ -52,3 +70,51 @@ class ImportHistory(Base):
|
|||||||
question_count: Mapped[int] = mapped_column(Integer, default=0)
|
question_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
status: Mapped[str] = mapped_column(Text, default="success")
|
status: Mapped[str] = mapped_column(Text, default="success")
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportJob(Base):
|
||||||
|
"""导入任务(持久化队列项)。"""
|
||||||
|
|
||||||
|
__tablename__ = "import_jobs"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), default=JOB_STATUS_QUEUED, index=True)
|
||||||
|
method: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
total: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
processed: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
success_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
failed_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
current_index: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
current_file: Mapped[str] = mapped_column(Text, default="")
|
||||||
|
error: Mapped[str] = mapped_column(Text, default="")
|
||||||
|
attempt: Mapped[int] = mapped_column(Integer, default=1)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
||||||
|
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||||
|
ended_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow
|
||||||
|
)
|
||||||
|
|
||||||
|
items: Mapped[list["ImportJobItem"]] = relationship(
|
||||||
|
"ImportJobItem", back_populates="job", order_by="ImportJobItem.seq", lazy="selectin"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportJobItem(Base):
|
||||||
|
"""导入任务内的单个文件项。"""
|
||||||
|
|
||||||
|
__tablename__ = "import_job_items"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||||
|
job_id: Mapped[int] = mapped_column(ForeignKey("import_jobs.id", ondelete="CASCADE"), nullable=False)
|
||||||
|
seq: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
filename: Mapped[str] = mapped_column(Text, default="")
|
||||||
|
stored_path: Mapped[str] = mapped_column(Text, default="")
|
||||||
|
status: Mapped[str] = mapped_column(String(32), default=JOB_STATUS_QUEUED)
|
||||||
|
attempt: Mapped[int] = mapped_column(Integer, default=1)
|
||||||
|
error: Mapped[str] = mapped_column(Text, default="")
|
||||||
|
question_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||||
|
ended_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||||
|
|
||||||
|
job: Mapped["ImportJob"] = relationship("ImportJob", back_populates="items")
|
||||||
|
|||||||
1
backend/repositories/__init__.py
Normal file
1
backend/repositories/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
# Repositories for DB access
|
||||||
181
backend/repositories/import_job_repo.py
Normal file
181
backend/repositories/import_job_repo.py
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
"""Import job persistence: create, get, list, update, claim for FIFO worker."""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from backend.models import (
|
||||||
|
JOB_STATUS_FAILED,
|
||||||
|
JOB_STATUS_QUEUED,
|
||||||
|
JOB_STATUS_RUNNING,
|
||||||
|
ImportJob,
|
||||||
|
ImportJobItem,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_job(
|
||||||
|
db: Session,
|
||||||
|
method: str,
|
||||||
|
items: list[dict],
|
||||||
|
) -> ImportJob:
|
||||||
|
"""Create a new import job with items. items: [{"filename": str, "stored_path": str}, ...]."""
|
||||||
|
job = ImportJob(
|
||||||
|
status=JOB_STATUS_QUEUED,
|
||||||
|
method=method,
|
||||||
|
total=len(items),
|
||||||
|
processed=0,
|
||||||
|
success_count=0,
|
||||||
|
failed_count=0,
|
||||||
|
)
|
||||||
|
db.add(job)
|
||||||
|
db.flush()
|
||||||
|
for seq, it in enumerate(items, start=1):
|
||||||
|
db.add(
|
||||||
|
ImportJobItem(
|
||||||
|
job_id=job.id,
|
||||||
|
seq=seq,
|
||||||
|
filename=it.get("filename", ""),
|
||||||
|
stored_path=it.get("stored_path", ""),
|
||||||
|
status=JOB_STATUS_QUEUED,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(job)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def create_job_empty(db: Session, method: str) -> ImportJob:
|
||||||
|
"""Create job with no items; caller then adds items and sets total."""
|
||||||
|
job = ImportJob(
|
||||||
|
status=JOB_STATUS_QUEUED,
|
||||||
|
method=method,
|
||||||
|
total=0,
|
||||||
|
processed=0,
|
||||||
|
success_count=0,
|
||||||
|
failed_count=0,
|
||||||
|
)
|
||||||
|
db.add(job)
|
||||||
|
db.flush()
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def add_job_items(db: Session, job_id: int, items: list[dict]) -> None:
|
||||||
|
"""Append items to job and set job.total. items: [{"filename": str, "stored_path": str}, ...]."""
|
||||||
|
job = db.query(ImportJob).filter(ImportJob.id == job_id).first()
|
||||||
|
if not job:
|
||||||
|
return
|
||||||
|
for seq, it in enumerate(items, start=1):
|
||||||
|
db.add(
|
||||||
|
ImportJobItem(
|
||||||
|
job_id=job_id,
|
||||||
|
seq=seq,
|
||||||
|
filename=it.get("filename", ""),
|
||||||
|
stored_path=it.get("stored_path", ""),
|
||||||
|
status=JOB_STATUS_QUEUED,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
job.total = len(items)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def get_job(db: Session, job_id: int) -> ImportJob | None:
|
||||||
|
"""Get job by id with items loaded."""
|
||||||
|
return db.query(ImportJob).filter(ImportJob.id == job_id).first()
|
||||||
|
|
||||||
|
|
||||||
|
def list_jobs(
|
||||||
|
db: Session,
|
||||||
|
statuses: list[str] | None = None,
|
||||||
|
limit: int = 50,
|
||||||
|
) -> list[ImportJob]:
|
||||||
|
"""List jobs, optionally filtered by status(es), newest first."""
|
||||||
|
q = db.query(ImportJob).order_by(ImportJob.created_at.desc())
|
||||||
|
if statuses:
|
||||||
|
q = q.filter(ImportJob.status.in_(statuses))
|
||||||
|
return q.limit(limit).all()
|
||||||
|
|
||||||
|
|
||||||
|
def update_job(db: Session, job_id: int, **kwargs) -> ImportJob | None:
|
||||||
|
"""Update job fields. Returns updated job or None if not found."""
|
||||||
|
job = db.query(ImportJob).filter(ImportJob.id == job_id).first()
|
||||||
|
if not job:
|
||||||
|
return None
|
||||||
|
for k, v in kwargs.items():
|
||||||
|
if hasattr(job, k):
|
||||||
|
setattr(job, k, v)
|
||||||
|
job.updated_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
db.refresh(job)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def update_job_item(db: Session, item_id: int, **kwargs) -> ImportJobItem | None:
|
||||||
|
"""Update a single job item."""
|
||||||
|
item = db.query(ImportJobItem).filter(ImportJobItem.id == item_id).first()
|
||||||
|
if not item:
|
||||||
|
return None
|
||||||
|
for k, v in kwargs.items():
|
||||||
|
if hasattr(item, k):
|
||||||
|
setattr(item, k, v)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(item)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def claim_oldest_queued(db: Session) -> ImportJob | None:
|
||||||
|
"""
|
||||||
|
Claim the oldest queued job by setting status to running.
|
||||||
|
Returns the job if claimed, None if no queued job.
|
||||||
|
Used by single-worker FIFO loop. (SQLite-compatible: no row locking.)
|
||||||
|
"""
|
||||||
|
job = (
|
||||||
|
db.query(ImportJob)
|
||||||
|
.filter(ImportJob.status == JOB_STATUS_QUEUED)
|
||||||
|
.order_by(ImportJob.created_at.asc())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if not job:
|
||||||
|
return None
|
||||||
|
job.status = JOB_STATUS_RUNNING
|
||||||
|
job.started_at = datetime.utcnow()
|
||||||
|
job.updated_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
db.refresh(job)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def get_queued_job_for_worker(db: Session) -> ImportJob | None:
|
||||||
|
"""
|
||||||
|
Get oldest queued job without locking (SQLite has limited FOR UPDATE support).
|
||||||
|
Caller must then update status to running in same or follow-up transaction.
|
||||||
|
"""
|
||||||
|
job = (
|
||||||
|
db.query(ImportJob)
|
||||||
|
.filter(ImportJob.status == JOB_STATUS_QUEUED)
|
||||||
|
.order_by(ImportJob.created_at.asc())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def cancel_job(db: Session, job_id: int) -> ImportJob | None:
|
||||||
|
"""Set job status to cancelled if it is queued or running."""
|
||||||
|
job = db.query(ImportJob).filter(ImportJob.id == job_id).first()
|
||||||
|
if not job or job.status not in (JOB_STATUS_QUEUED, JOB_STATUS_RUNNING):
|
||||||
|
return None
|
||||||
|
job.status = "cancelled"
|
||||||
|
job.ended_at = datetime.utcnow()
|
||||||
|
job.updated_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
db.refresh(job)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def get_failed_items(db: Session, job_id: int) -> list[ImportJobItem]:
|
||||||
|
"""Return items with status failed for retry."""
|
||||||
|
return (
|
||||||
|
db.query(ImportJobItem)
|
||||||
|
.filter(ImportJobItem.job_id == job_id, ImportJobItem.status == JOB_STATUS_FAILED)
|
||||||
|
.order_by(ImportJobItem.seq.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
@@ -10,3 +10,4 @@ python-multipart
|
|||||||
requests
|
requests
|
||||||
openpyxl
|
openpyxl
|
||||||
python-dotenv
|
python-dotenv
|
||||||
|
pytest
|
||||||
|
|||||||
@@ -1,25 +1,153 @@
|
|||||||
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from threading import Lock
|
||||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
from typing import Literal
|
||||||
|
from uuid import uuid4
|
||||||
|
from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, HTTPException, Query, UploadFile
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from backend.auth import get_current_user
|
from backend.auth import get_current_user
|
||||||
from backend.database import get_db
|
from backend.database import SessionLocal, get_db
|
||||||
from backend.models import ImportHistory, Question
|
from backend.models import ImportHistory, Question
|
||||||
from backend.schemas import ImportHistoryOut, QuestionOut
|
from backend.repositories import import_job_repo
|
||||||
|
from backend.schemas import ImportHistoryOut, ImportJobOut, QuestionOut
|
||||||
from backend.services.excel_service import create_template_bytes, parse_excel_file
|
from backend.services.excel_service import create_template_bytes, parse_excel_file
|
||||||
from backend.services.file_utils import save_upload
|
from backend.services.file_utils import save_upload, save_upload_for_job
|
||||||
from backend.services.parser import DMXAPIService, extract_metadata
|
from backend.services.parser import OpenAICompatibleParserService, extract_metadata
|
||||||
|
|
||||||
router = APIRouter(dependencies=[Depends(get_current_user)])
|
router = APIRouter(dependencies=[Depends(get_current_user)])
|
||||||
|
|
||||||
|
BATCH_TASKS: dict[str, dict] = {}
|
||||||
|
BATCH_TASKS_LOCK = Lock()
|
||||||
|
BatchMethod = Literal["excel", "ai"]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _set_task(task_id: str, **fields) -> None:
|
||||||
|
with BATCH_TASKS_LOCK:
|
||||||
|
task = BATCH_TASKS.get(task_id)
|
||||||
|
if not task:
|
||||||
|
return
|
||||||
|
task.update(fields)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_ai_rows(path: Path) -> list[dict]:
|
||||||
|
parser = OpenAICompatibleParserService()
|
||||||
|
metadata = extract_metadata(path.name)
|
||||||
|
questions = parser.parse_file(str(path))
|
||||||
|
rows = []
|
||||||
|
for q in questions:
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"chapter": metadata["chapter"],
|
||||||
|
"primary_knowledge": "",
|
||||||
|
"secondary_knowledge": metadata["secondary_knowledge"],
|
||||||
|
"question_type": metadata["question_type"],
|
||||||
|
"difficulty": metadata["difficulty"],
|
||||||
|
"stem": q.get("题干", ""),
|
||||||
|
"option_a": q.get("选项A", ""),
|
||||||
|
"option_b": q.get("选项B", ""),
|
||||||
|
"option_c": q.get("选项C", ""),
|
||||||
|
"option_d": q.get("选项D", ""),
|
||||||
|
"answer": q.get("正确答案", ""),
|
||||||
|
"explanation": q.get("解析", ""),
|
||||||
|
"notes": q.get("备注", ""),
|
||||||
|
"source_file": metadata["source_file"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _run_batch_import(task_id: str, files: list[dict], method: BatchMethod) -> None:
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
for index, item in enumerate(files, start=1):
|
||||||
|
path = Path(item["path"])
|
||||||
|
filename = item["filename"]
|
||||||
|
_set_task(task_id, current_file=filename)
|
||||||
|
try:
|
||||||
|
if method == "excel":
|
||||||
|
if path.suffix.lower() not in [".xlsx", ".xlsm", ".xltx", ".xltm"]:
|
||||||
|
raise ValueError("仅支持 Excel 文件")
|
||||||
|
rows = parse_excel_file(path)
|
||||||
|
else:
|
||||||
|
rows = _build_ai_rows(path)
|
||||||
|
|
||||||
|
questions = [Question(**row) for row in rows]
|
||||||
|
if questions:
|
||||||
|
db.add_all(questions)
|
||||||
|
db.add(
|
||||||
|
ImportHistory(
|
||||||
|
filename=filename,
|
||||||
|
method=method,
|
||||||
|
question_count=len(questions),
|
||||||
|
status="success",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
with BATCH_TASKS_LOCK:
|
||||||
|
task = BATCH_TASKS[task_id]
|
||||||
|
task["success_count"] += 1
|
||||||
|
task["total_questions"] += len(questions)
|
||||||
|
task["results"].append(
|
||||||
|
{
|
||||||
|
"filename": filename,
|
||||||
|
"status": "success",
|
||||||
|
"question_count": len(questions),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
db.rollback()
|
||||||
|
db.add(
|
||||||
|
ImportHistory(
|
||||||
|
filename=filename,
|
||||||
|
method=method,
|
||||||
|
question_count=0,
|
||||||
|
status="failed",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
with BATCH_TASKS_LOCK:
|
||||||
|
task = BATCH_TASKS[task_id]
|
||||||
|
task["failed_count"] += 1
|
||||||
|
task["results"].append(
|
||||||
|
{
|
||||||
|
"filename": filename,
|
||||||
|
"status": "failed",
|
||||||
|
"error": str(exc),
|
||||||
|
"question_count": 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
_set_task(task_id, processed=index)
|
||||||
|
|
||||||
|
_set_task(
|
||||||
|
task_id,
|
||||||
|
status="completed",
|
||||||
|
current_file="",
|
||||||
|
ended_at=datetime.utcnow().isoformat(),
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
_set_task(
|
||||||
|
task_id,
|
||||||
|
status="failed",
|
||||||
|
error=str(exc),
|
||||||
|
ended_at=datetime.utcnow().isoformat(),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
@router.post("/ai/parse")
|
@router.post("/ai/parse")
|
||||||
def parse_by_ai(file: UploadFile = File(...)) -> dict:
|
def parse_by_ai(file: UploadFile = File(...)) -> dict:
|
||||||
path = save_upload(file)
|
path = save_upload(file)
|
||||||
parser = DMXAPIService()
|
parser = OpenAICompatibleParserService()
|
||||||
metadata = extract_metadata(file.filename or path.name)
|
metadata = extract_metadata(file.filename or path.name)
|
||||||
questions = parser.parse_file(str(path))
|
try:
|
||||||
|
questions = parser.parse_file(str(path))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||||
preview = []
|
preview = []
|
||||||
for q in questions:
|
for q in questions:
|
||||||
preview.append(
|
preview.append(
|
||||||
@@ -95,3 +223,110 @@ def download_template() -> dict:
|
|||||||
def import_history(db: Session = Depends(get_db)) -> list[ImportHistoryOut]:
|
def import_history(db: Session = Depends(get_db)) -> list[ImportHistoryOut]:
|
||||||
rows = db.query(ImportHistory).order_by(ImportHistory.created_at.desc()).limit(100).all()
|
rows = db.query(ImportHistory).order_by(ImportHistory.created_at.desc()).limit(100).all()
|
||||||
return [ImportHistoryOut.model_validate(r) for r in rows]
|
return [ImportHistoryOut.model_validate(r) for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/batch/start")
|
||||||
|
def start_batch_import(
|
||||||
|
background_tasks: BackgroundTasks,
|
||||||
|
files: list[UploadFile] = File(...),
|
||||||
|
method: BatchMethod = Form("ai"),
|
||||||
|
) -> dict:
|
||||||
|
if not files:
|
||||||
|
raise HTTPException(status_code=400, detail="请至少上传一个文件")
|
||||||
|
|
||||||
|
task_id = uuid4().hex
|
||||||
|
saved_files: list[dict] = []
|
||||||
|
for f in files:
|
||||||
|
saved_path = save_upload(f)
|
||||||
|
saved_files.append({"path": str(saved_path), "filename": f.filename or Path(saved_path).name})
|
||||||
|
|
||||||
|
with BATCH_TASKS_LOCK:
|
||||||
|
BATCH_TASKS[task_id] = {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "running",
|
||||||
|
"method": method,
|
||||||
|
"total": len(saved_files),
|
||||||
|
"processed": 0,
|
||||||
|
"current_file": "",
|
||||||
|
"success_count": 0,
|
||||||
|
"failed_count": 0,
|
||||||
|
"total_questions": 0,
|
||||||
|
"results": [],
|
||||||
|
"error": "",
|
||||||
|
"started_at": datetime.utcnow().isoformat(),
|
||||||
|
"ended_at": "",
|
||||||
|
}
|
||||||
|
background_tasks.add_task(_run_batch_import, task_id, saved_files, method)
|
||||||
|
return {"task_id": task_id}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/batch/{task_id}")
|
||||||
|
def get_batch_import_progress(task_id: str) -> dict:
|
||||||
|
with BATCH_TASKS_LOCK:
|
||||||
|
task = BATCH_TASKS.get(task_id)
|
||||||
|
if not task:
|
||||||
|
raise HTTPException(status_code=404, detail="任务不存在")
|
||||||
|
total = task["total"] or 1
|
||||||
|
progress = round((task["processed"] / total) * 100, 2)
|
||||||
|
return {**task, "progress": progress}
|
||||||
|
|
||||||
|
|
||||||
|
# ----- 持久化队列 Jobs API -----
|
||||||
|
|
||||||
|
@router.post("/jobs", response_model=ImportJobOut)
|
||||||
|
def create_import_job(
|
||||||
|
files: list[UploadFile] = File(...),
|
||||||
|
method: BatchMethod = Form("ai"),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> ImportJobOut:
|
||||||
|
if not files:
|
||||||
|
raise HTTPException(status_code=400, detail="请至少上传一个文件")
|
||||||
|
job = import_job_repo.create_job_empty(db, method)
|
||||||
|
items: list[dict] = []
|
||||||
|
for seq, f in enumerate(files, start=1):
|
||||||
|
path = save_upload_for_job(job.id, seq, f)
|
||||||
|
items.append({"filename": f.filename or path.name, "stored_path": str(path)})
|
||||||
|
import_job_repo.add_job_items(db, job.id, items)
|
||||||
|
job = import_job_repo.get_job(db, job.id)
|
||||||
|
return ImportJobOut.model_validate(job)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/jobs/{job_id}", response_model=ImportJobOut)
|
||||||
|
def get_import_job(job_id: int, db: Session = Depends(get_db)) -> ImportJobOut:
|
||||||
|
job = import_job_repo.get_job(db, job_id)
|
||||||
|
if not job:
|
||||||
|
raise HTTPException(status_code=404, detail="任务不存在")
|
||||||
|
return ImportJobOut.model_validate(job)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/jobs", response_model=list[ImportJobOut])
|
||||||
|
def list_import_jobs(
|
||||||
|
status: str | None = Query(None, description="queued,running 等,逗号分隔"),
|
||||||
|
limit: int = Query(50, le=100),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> list[ImportJobOut]:
|
||||||
|
statuses = [s.strip() for s in status.split(",") if s.strip()] if status else None
|
||||||
|
jobs = import_job_repo.list_jobs(db, statuses=statuses, limit=limit)
|
||||||
|
return [ImportJobOut.model_validate(j) for j in jobs]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/jobs/{job_id}/retry", response_model=ImportJobOut)
|
||||||
|
def retry_import_job(job_id: int, db: Session = Depends(get_db)) -> ImportJobOut:
|
||||||
|
job = import_job_repo.get_job(db, job_id)
|
||||||
|
if not job:
|
||||||
|
raise HTTPException(status_code=404, detail="任务不存在")
|
||||||
|
failed = import_job_repo.get_failed_items(db, job_id)
|
||||||
|
if not failed:
|
||||||
|
raise HTTPException(status_code=400, detail="没有可重试的失败项")
|
||||||
|
items = [{"filename": it.filename, "stored_path": it.stored_path} for it in failed]
|
||||||
|
new_job = import_job_repo.create_job(db, job.method, items)
|
||||||
|
new_job = import_job_repo.get_job(db, new_job.id)
|
||||||
|
return ImportJobOut.model_validate(new_job)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/jobs/{job_id}/cancel", response_model=ImportJobOut)
|
||||||
|
def cancel_import_job(job_id: int, db: Session = Depends(get_db)) -> ImportJobOut:
|
||||||
|
job = import_job_repo.cancel_job(db, job_id)
|
||||||
|
if not job:
|
||||||
|
raise HTTPException(status_code=400, detail="任务不存在或无法取消")
|
||||||
|
return ImportJobOut.model_validate(job)
|
||||||
|
|||||||
@@ -103,6 +103,45 @@ class ImportHistoryOut(BaseModel):
|
|||||||
from_attributes = True
|
from_attributes = True
|
||||||
|
|
||||||
|
|
||||||
|
class ImportJobItemOut(BaseModel):
|
||||||
|
id: int
|
||||||
|
job_id: int
|
||||||
|
seq: int
|
||||||
|
filename: str
|
||||||
|
stored_path: str
|
||||||
|
status: str
|
||||||
|
attempt: int
|
||||||
|
error: str
|
||||||
|
question_count: int
|
||||||
|
started_at: datetime | None
|
||||||
|
ended_at: datetime | None
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
from_attributes = True
|
||||||
|
|
||||||
|
|
||||||
|
class ImportJobOut(BaseModel):
|
||||||
|
id: int
|
||||||
|
status: str
|
||||||
|
method: str
|
||||||
|
total: int
|
||||||
|
processed: int
|
||||||
|
success_count: int
|
||||||
|
failed_count: int
|
||||||
|
current_index: int
|
||||||
|
current_file: str
|
||||||
|
error: str
|
||||||
|
attempt: int
|
||||||
|
created_at: datetime
|
||||||
|
started_at: datetime | None
|
||||||
|
ended_at: datetime | None
|
||||||
|
updated_at: datetime
|
||||||
|
items: list[ImportJobItemOut] = []
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
from_attributes = True
|
||||||
|
|
||||||
|
|
||||||
class PracticeStartRequest(BaseModel):
|
class PracticeStartRequest(BaseModel):
|
||||||
chapter: Optional[str] = None
|
chapter: Optional[str] = None
|
||||||
secondary_knowledge: Optional[str] = None
|
secondary_knowledge: Optional[str] = None
|
||||||
|
|||||||
@@ -18,3 +18,14 @@ def save_upload(upload_file: UploadFile) -> Path:
|
|||||||
with target_path.open("wb") as buffer:
|
with target_path.open("wb") as buffer:
|
||||||
shutil.copyfileobj(upload_file.file, buffer)
|
shutil.copyfileobj(upload_file.file, buffer)
|
||||||
return target_path
|
return target_path
|
||||||
|
|
||||||
|
|
||||||
|
def save_upload_for_job(job_id: int, seq: int, upload_file: UploadFile) -> Path:
|
||||||
|
"""Save file with unique path under upload_dir/job_id/ to avoid overwrites."""
|
||||||
|
base = ensure_upload_dir() / str(job_id)
|
||||||
|
base.mkdir(parents=True, exist_ok=True)
|
||||||
|
name = upload_file.filename or "file"
|
||||||
|
target_path = base / f"{seq}_{name}"
|
||||||
|
with target_path.open("wb") as buffer:
|
||||||
|
shutil.copyfileobj(upload_file.file, buffer)
|
||||||
|
return target_path
|
||||||
|
|||||||
180
backend/services/import_queue_service.py
Normal file
180
backend/services/import_queue_service.py
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
"""
|
||||||
|
Import queue: single-consumer FIFO worker and job execution.
|
||||||
|
Run run_worker_loop() in a background thread; on startup call reset_stale_running_jobs().
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
import time
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from backend.database import SessionLocal
|
||||||
|
from backend.models import (
|
||||||
|
JOB_STATUS_FAILED,
|
||||||
|
JOB_STATUS_QUEUED,
|
||||||
|
JOB_STATUS_RUNNING,
|
||||||
|
JOB_STATUS_SUCCESS,
|
||||||
|
ImportHistory,
|
||||||
|
ImportJob,
|
||||||
|
ImportJobItem,
|
||||||
|
Question,
|
||||||
|
)
|
||||||
|
from backend.repositories import import_job_repo as repo
|
||||||
|
from backend.services.excel_service import parse_excel_file
|
||||||
|
from backend.services.parser import OpenAICompatibleParserService, extract_metadata
|
||||||
|
|
||||||
|
|
||||||
|
def _build_ai_rows(path: Path) -> list[dict]:
|
||||||
|
parser = OpenAICompatibleParserService()
|
||||||
|
metadata = extract_metadata(path.name)
|
||||||
|
questions = parser.parse_file(str(path))
|
||||||
|
rows = []
|
||||||
|
for q in questions:
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"chapter": metadata["chapter"],
|
||||||
|
"primary_knowledge": "",
|
||||||
|
"secondary_knowledge": metadata["secondary_knowledge"],
|
||||||
|
"question_type": metadata["question_type"],
|
||||||
|
"difficulty": metadata["difficulty"],
|
||||||
|
"stem": q.get("题干", ""),
|
||||||
|
"option_a": q.get("选项A", ""),
|
||||||
|
"option_b": q.get("选项B", ""),
|
||||||
|
"option_c": q.get("选项C", ""),
|
||||||
|
"option_d": q.get("选项D", ""),
|
||||||
|
"answer": q.get("正确答案", ""),
|
||||||
|
"explanation": q.get("解析", ""),
|
||||||
|
"notes": q.get("备注", ""),
|
||||||
|
"source_file": metadata["source_file"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _process_one_item(
|
||||||
|
db: Session,
|
||||||
|
job: ImportJob,
|
||||||
|
item: ImportJobItem,
|
||||||
|
method: str,
|
||||||
|
) -> None:
|
||||||
|
path = Path(item.stored_path)
|
||||||
|
filename = item.filename
|
||||||
|
job.current_file = filename
|
||||||
|
job.current_index = item.seq
|
||||||
|
job.updated_at = datetime.utcnow()
|
||||||
|
item.status = JOB_STATUS_RUNNING
|
||||||
|
item.started_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
try:
|
||||||
|
if method == "excel":
|
||||||
|
if path.suffix.lower() not in [".xlsx", ".xlsm", ".xltx", ".xltm"]:
|
||||||
|
raise ValueError("仅支持 Excel 文件")
|
||||||
|
rows = parse_excel_file(path)
|
||||||
|
else:
|
||||||
|
rows = _build_ai_rows(path)
|
||||||
|
|
||||||
|
questions = [Question(**row) for row in rows]
|
||||||
|
if questions:
|
||||||
|
db.add_all(questions)
|
||||||
|
db.add(
|
||||||
|
ImportHistory(
|
||||||
|
filename=filename,
|
||||||
|
method=method,
|
||||||
|
question_count=len(questions),
|
||||||
|
status="success",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
item.status = JOB_STATUS_SUCCESS
|
||||||
|
item.question_count = len(questions)
|
||||||
|
item.ended_at = datetime.utcnow()
|
||||||
|
job.success_count += 1
|
||||||
|
job.processed += 1
|
||||||
|
job.updated_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
except Exception as exc:
|
||||||
|
db.rollback()
|
||||||
|
db.add(
|
||||||
|
ImportHistory(
|
||||||
|
filename=filename,
|
||||||
|
method=method,
|
||||||
|
question_count=0,
|
||||||
|
status="failed",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
item.status = JOB_STATUS_FAILED
|
||||||
|
item.error = str(exc)
|
||||||
|
item.ended_at = datetime.utcnow()
|
||||||
|
job.failed_count += 1
|
||||||
|
job.processed += 1
|
||||||
|
job.updated_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def process_job(db: Session, job_id: int) -> None:
|
||||||
|
"""Execute a single job: process all items in order, then set job terminal status."""
|
||||||
|
job = repo.get_job(db, job_id)
|
||||||
|
if not job or job.status != JOB_STATUS_RUNNING:
|
||||||
|
return
|
||||||
|
method = job.method
|
||||||
|
items = sorted(job.items, key=lambda x: x.seq)
|
||||||
|
# Resume: ensure processed/success_count/failed_count reflect already-completed items
|
||||||
|
job.processed = sum(1 for it in items if it.status in (JOB_STATUS_SUCCESS, JOB_STATUS_FAILED))
|
||||||
|
job.success_count = sum(1 for it in items if it.status == JOB_STATUS_SUCCESS)
|
||||||
|
job.failed_count = sum(1 for it in items if it.status == JOB_STATUS_FAILED)
|
||||||
|
db.commit()
|
||||||
|
for item in items:
|
||||||
|
if item.status in (JOB_STATUS_SUCCESS, JOB_STATUS_FAILED):
|
||||||
|
continue
|
||||||
|
_process_one_item(db, job, item, method)
|
||||||
|
db.refresh(job)
|
||||||
|
|
||||||
|
job = repo.get_job(db, job_id)
|
||||||
|
if not job:
|
||||||
|
return
|
||||||
|
if job.failed_count > 0 and job.success_count == 0:
|
||||||
|
job.status = JOB_STATUS_FAILED
|
||||||
|
job.error = "部分或全部文件处理失败"
|
||||||
|
else:
|
||||||
|
job.status = JOB_STATUS_SUCCESS
|
||||||
|
job.error = ""
|
||||||
|
job.ended_at = datetime.utcnow()
|
||||||
|
job.current_file = ""
|
||||||
|
job.updated_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def reset_stale_running_jobs(db: Session) -> int:
|
||||||
|
"""On startup: set any job left in 'running' back to 'queued' so worker can pick it up."""
|
||||||
|
count = 0
|
||||||
|
for job in db.query(ImportJob).filter(ImportJob.status == JOB_STATUS_RUNNING).all():
|
||||||
|
job.status = JOB_STATUS_QUEUED
|
||||||
|
count += 1
|
||||||
|
if count:
|
||||||
|
db.commit()
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
def run_worker_loop(interval_seconds: float = 1.0) -> None:
|
||||||
|
"""
|
||||||
|
Single-consumer FIFO loop. Call from a background thread.
|
||||||
|
Claims oldest queued job, processes it, then repeats. Sleeps when no job.
|
||||||
|
"""
|
||||||
|
while True:
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
job = repo.claim_oldest_queued(db)
|
||||||
|
if job:
|
||||||
|
process_job(db, job.id)
|
||||||
|
else:
|
||||||
|
time.sleep(interval_seconds)
|
||||||
|
except Exception:
|
||||||
|
if db:
|
||||||
|
db.rollback()
|
||||||
|
time.sleep(interval_seconds)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
@@ -8,11 +10,13 @@ import requests
|
|||||||
from backend.config import settings
|
from backend.config import settings
|
||||||
|
|
||||||
|
|
||||||
class DMXAPIService:
|
|
||||||
|
|
||||||
|
class OpenAICompatibleParserService:
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.api_key = settings.api_key
|
self.api_key = settings.api_key
|
||||||
self.model_name = settings.model_name
|
self.model_name = settings.model_name
|
||||||
self.api_url = settings.dmxapi_url
|
self.api_url = settings.openai_api_url
|
||||||
|
|
||||||
def parse_file(self, file_path: str) -> list[dict]:
|
def parse_file(self, file_path: str) -> list[dict]:
|
||||||
path = Path(file_path)
|
path = Path(file_path)
|
||||||
@@ -51,33 +55,94 @@ class DMXAPIService:
|
|||||||
def _convert_to_pdf(self, path: Path) -> Path:
|
def _convert_to_pdf(self, path: Path) -> Path:
|
||||||
pdf_path = path.with_suffix(".pdf")
|
pdf_path = path.with_suffix(".pdf")
|
||||||
pdf_path.unlink(missing_ok=True)
|
pdf_path.unlink(missing_ok=True)
|
||||||
|
source_path = path
|
||||||
|
temp_dir: str | None = None
|
||||||
|
|
||||||
cmd = [
|
if path.suffix.lower() == ".doc":
|
||||||
"pandoc",
|
source_path, temp_dir = self._convert_doc_to_docx(path)
|
||||||
str(path),
|
|
||||||
"-o",
|
try:
|
||||||
str(pdf_path),
|
cmd = [
|
||||||
"--pdf-engine=xelatex",
|
|
||||||
"-V",
|
|
||||||
"CJKmainfont=PingFang SC",
|
|
||||||
]
|
|
||||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=90)
|
|
||||||
if result.returncode != 0:
|
|
||||||
fallback = [
|
|
||||||
"pandoc",
|
"pandoc",
|
||||||
str(path),
|
str(source_path),
|
||||||
"-o",
|
"-o",
|
||||||
str(pdf_path),
|
str(pdf_path),
|
||||||
"--pdf-engine=weasyprint",
|
"--pdf-engine=xelatex",
|
||||||
|
"-V",
|
||||||
|
"CJKmainfont=PingFang SC",
|
||||||
]
|
]
|
||||||
result = subprocess.run(fallback, capture_output=True, text=True, timeout=90)
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=90)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
raise ValueError(f"文件转 PDF 失败: {result.stderr}")
|
fallback = [
|
||||||
return pdf_path
|
"pandoc",
|
||||||
|
str(source_path),
|
||||||
|
"-o",
|
||||||
|
str(pdf_path),
|
||||||
|
"--pdf-engine=weasyprint",
|
||||||
|
]
|
||||||
|
result = subprocess.run(fallback, capture_output=True, text=True, timeout=90)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise ValueError(f"文件转 PDF 失败: {result.stderr}")
|
||||||
|
return pdf_path
|
||||||
|
finally:
|
||||||
|
if temp_dir:
|
||||||
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||||
|
|
||||||
|
def _convert_doc_to_docx(self, path: Path) -> tuple[Path, str]:
|
||||||
|
temp_dir = tempfile.mkdtemp(prefix="pb_doc_convert_")
|
||||||
|
converted_path = Path(temp_dir) / f"{path.stem}.docx"
|
||||||
|
convert_errors: list[str] = []
|
||||||
|
|
||||||
|
if shutil.which("soffice"):
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"soffice",
|
||||||
|
"--headless",
|
||||||
|
"--convert-to",
|
||||||
|
"docx",
|
||||||
|
"--outdir",
|
||||||
|
temp_dir,
|
||||||
|
str(path),
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
if result.returncode == 0 and converted_path.exists():
|
||||||
|
return converted_path, temp_dir
|
||||||
|
convert_errors.append(f"soffice: {(result.stderr or result.stdout).strip()}")
|
||||||
|
else:
|
||||||
|
convert_errors.append("soffice: 未安装")
|
||||||
|
|
||||||
|
if shutil.which("textutil"):
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"textutil",
|
||||||
|
"-convert",
|
||||||
|
"docx",
|
||||||
|
"-output",
|
||||||
|
str(converted_path),
|
||||||
|
str(path),
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
if result.returncode == 0 and converted_path.exists():
|
||||||
|
return converted_path, temp_dir
|
||||||
|
convert_errors.append(f"textutil: {(result.stderr or result.stdout).strip()}")
|
||||||
|
else:
|
||||||
|
convert_errors.append("textutil: 未安装")
|
||||||
|
|
||||||
|
raise ValueError(
|
||||||
|
"检测到 .doc 文件,pandoc 不支持直接转换。"
|
||||||
|
"已尝试自动转换为 .docx 但失败。请先把 .doc 另存为 .docx 后重试。"
|
||||||
|
f" 详细信息: {' | '.join(convert_errors)}"
|
||||||
|
)
|
||||||
|
|
||||||
def _parse_with_file_url(self, file_url: str, original_filename: str) -> list[dict]:
|
def _parse_with_file_url(self, file_url: str, original_filename: str) -> list[dict]:
|
||||||
if not self.api_key:
|
if not self.api_key:
|
||||||
raise ValueError("未配置 API_KEY,无法调用 DMXAPI")
|
raise ValueError("未配置 API_KEY,无法调用 OpenAI 兼容接口")
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"model": self.model_name,
|
"model": self.model_name,
|
||||||
@@ -99,7 +164,7 @@ class DMXAPIService:
|
|||||||
self.api_url, headers=headers, data=json.dumps(payload), timeout=180
|
self.api_url, headers=headers, data=json.dumps(payload), timeout=180
|
||||||
)
|
)
|
||||||
if response.status_code != 200:
|
if response.status_code != 200:
|
||||||
raise ValueError(f"DMXAPI 请求失败: {response.status_code} {response.text}")
|
raise ValueError(f"OpenAI 兼容接口请求失败: {response.status_code} {response.text}")
|
||||||
return self._extract_questions(response.json())
|
return self._extract_questions(response.json())
|
||||||
|
|
||||||
def _build_instruction(self, filename: str) -> str:
|
def _build_instruction(self, filename: str) -> str:
|
||||||
|
|||||||
1
backend/tests/__init__.py
Normal file
1
backend/tests/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
# Backend tests
|
||||||
17
backend/tests/conftest.py
Normal file
17
backend/tests/conftest.py
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
import pytest
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from backend.database import Base
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def db_session():
|
||||||
|
engine = create_engine("sqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(engine)
|
||||||
|
Session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||||
|
session = Session()
|
||||||
|
try:
|
||||||
|
yield session
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
82
backend/tests/test_import_job_repo.py
Normal file
82
backend/tests/test_import_job_repo.py
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
"""Tests for import job repository: create, get, list, claim FIFO, cancel, failed items."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from backend.models import JOB_STATUS_QUEUED, JOB_STATUS_RUNNING
|
||||||
|
from backend.repositories import import_job_repo as repo
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_job(db_session):
|
||||||
|
items = [
|
||||||
|
{"filename": "a.xlsx", "stored_path": "/up/1/a.xlsx"},
|
||||||
|
{"filename": "b.xlsx", "stored_path": "/up/1/b.xlsx"},
|
||||||
|
]
|
||||||
|
job = repo.create_job(db_session, "excel", items)
|
||||||
|
assert job.id is not None
|
||||||
|
assert job.status == JOB_STATUS_QUEUED
|
||||||
|
assert job.method == "excel"
|
||||||
|
assert job.total == 2
|
||||||
|
assert job.processed == 0
|
||||||
|
assert len(job.items) == 2
|
||||||
|
assert job.items[0].filename == "a.xlsx" and job.items[0].seq == 1
|
||||||
|
assert job.items[1].filename == "b.xlsx" and job.items[1].seq == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_job(db_session):
|
||||||
|
job = repo.create_job(db_session, "ai", [{"filename": "f.pdf", "stored_path": "/f.pdf"}])
|
||||||
|
loaded = repo.get_job(db_session, job.id)
|
||||||
|
assert loaded is not None
|
||||||
|
assert loaded.id == job.id
|
||||||
|
assert len(loaded.items) == 1
|
||||||
|
assert repo.get_job(db_session, 99999) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_jobs(db_session):
|
||||||
|
repo.create_job(db_session, "excel", [{"filename": "a", "stored_path": "/a"}])
|
||||||
|
repo.create_job(db_session, "excel", [{"filename": "b", "stored_path": "/b"}])
|
||||||
|
all_jobs = repo.list_jobs(db_session, limit=10)
|
||||||
|
assert len(all_jobs) >= 2
|
||||||
|
statuses = repo.list_jobs(db_session, statuses=[JOB_STATUS_QUEUED], limit=10)
|
||||||
|
assert all(j.status == JOB_STATUS_QUEUED for j in statuses)
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_oldest_queued_fifo(db_session):
|
||||||
|
j1 = repo.create_job(db_session, "excel", [{"filename": "a", "stored_path": "/a"}])
|
||||||
|
j2 = repo.create_job(db_session, "excel", [{"filename": "b", "stored_path": "/b"}])
|
||||||
|
claimed = repo.claim_oldest_queued(db_session)
|
||||||
|
assert claimed is not None
|
||||||
|
assert claimed.id == j1.id
|
||||||
|
assert claimed.status == JOB_STATUS_RUNNING
|
||||||
|
second = repo.claim_oldest_queued(db_session)
|
||||||
|
assert second is not None
|
||||||
|
assert second.id == j2.id
|
||||||
|
assert repo.claim_oldest_queued(db_session) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_cancel_job(db_session):
|
||||||
|
job = repo.create_job(db_session, "excel", [{"filename": "a", "stored_path": "/a"}])
|
||||||
|
cancelled = repo.cancel_job(db_session, job.id)
|
||||||
|
assert cancelled is not None
|
||||||
|
assert cancelled.status == "cancelled"
|
||||||
|
loaded = repo.get_job(db_session, job.id)
|
||||||
|
assert loaded.status == "cancelled"
|
||||||
|
assert repo.cancel_job(db_session, job.id) is None # already cancelled
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_failed_items(db_session):
|
||||||
|
job = repo.create_job(
|
||||||
|
db_session,
|
||||||
|
"excel",
|
||||||
|
[
|
||||||
|
{"filename": "a", "stored_path": "/a"},
|
||||||
|
{"filename": "b", "stored_path": "/b"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
failed = repo.get_failed_items(db_session, job.id)
|
||||||
|
assert len(failed) == 0
|
||||||
|
job.items[0].status = "failed"
|
||||||
|
job.items[0].error = "err"
|
||||||
|
db_session.commit()
|
||||||
|
failed = repo.get_failed_items(db_session, job.id)
|
||||||
|
assert len(failed) == 1
|
||||||
|
assert failed[0].filename == "a"
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { Layout, Menu, message } from "antd";
|
import { Layout, Menu, message } from "antd";
|
||||||
import { useMemo, useState } from "react";
|
import { useEffect, useMemo, useState } from "react";
|
||||||
import { Navigate, Route, Routes, useLocation, useNavigate } from "react-router-dom";
|
import { Navigate, Route, Routes, useLocation, useNavigate } from "react-router-dom";
|
||||||
import LoginModal from "./components/LoginModal";
|
import LoginModal from "./components/LoginModal";
|
||||||
import Categories from "./pages/Categories";
|
import Categories from "./pages/Categories";
|
||||||
@@ -26,6 +26,14 @@ export default function App() {
|
|||||||
const location = useLocation();
|
const location = useLocation();
|
||||||
const [loggedIn, setLoggedIn] = useState(Boolean(localStorage.getItem("pb_token")));
|
const [loggedIn, setLoggedIn] = useState(Boolean(localStorage.getItem("pb_token")));
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const handleUnauthorized = () => {
|
||||||
|
setLoggedIn(false);
|
||||||
|
};
|
||||||
|
window.addEventListener("pb-auth-unauthorized", handleUnauthorized);
|
||||||
|
return () => window.removeEventListener("pb-auth-unauthorized", handleUnauthorized);
|
||||||
|
}, []);
|
||||||
|
|
||||||
const selectedKey = useMemo(() => {
|
const selectedKey = useMemo(() => {
|
||||||
const hit = menuItems.find((m) => location.pathname.startsWith(m.key));
|
const hit = menuItems.find((m) => location.pathname.startsWith(m.key));
|
||||||
return hit ? [hit.key] : ["/dashboard"];
|
return hit ? [hit.key] : ["/dashboard"];
|
||||||
|
|||||||
@@ -12,4 +12,80 @@ api.interceptors.request.use((config) => {
|
|||||||
return config;
|
return config;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
api.interceptors.response.use(
|
||||||
|
(response) => response,
|
||||||
|
(error) => {
|
||||||
|
if (error?.response?.status === 401) {
|
||||||
|
localStorage.removeItem("pb_token");
|
||||||
|
window.dispatchEvent(new CustomEvent("pb-auth-unauthorized"));
|
||||||
|
}
|
||||||
|
return Promise.reject(error);
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
// 导入任务(持久化队列)类型与 API
|
||||||
|
export interface ImportJobItemOut {
|
||||||
|
id: number;
|
||||||
|
job_id: number;
|
||||||
|
seq: number;
|
||||||
|
filename: string;
|
||||||
|
stored_path: string;
|
||||||
|
status: string;
|
||||||
|
attempt: number;
|
||||||
|
error: string;
|
||||||
|
question_count: number;
|
||||||
|
started_at: string | null;
|
||||||
|
ended_at: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ImportJobOut {
|
||||||
|
id: number;
|
||||||
|
status: string;
|
||||||
|
method: string;
|
||||||
|
total: number;
|
||||||
|
processed: number;
|
||||||
|
success_count: number;
|
||||||
|
failed_count: number;
|
||||||
|
current_index: number;
|
||||||
|
current_file: string;
|
||||||
|
error: string;
|
||||||
|
attempt: number;
|
||||||
|
created_at: string;
|
||||||
|
started_at: string | null;
|
||||||
|
ended_at: string | null;
|
||||||
|
updated_at: string;
|
||||||
|
items: ImportJobItemOut[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createImportJob(files: File[], method: "excel" | "ai"): Promise<ImportJobOut> {
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append("method", method);
|
||||||
|
files.forEach((file) => formData.append("files", file));
|
||||||
|
const { data } = await api.post<ImportJobOut>("/import/jobs", formData, {
|
||||||
|
headers: { "Content-Type": "multipart/form-data" }
|
||||||
|
});
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getImportJob(jobId: number): Promise<ImportJobOut> {
|
||||||
|
const { data } = await api.get<ImportJobOut>(`/import/jobs/${jobId}`);
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listImportJobs(status?: string): Promise<ImportJobOut[]> {
|
||||||
|
const params = status ? { status } : {};
|
||||||
|
const { data } = await api.get<ImportJobOut[]>("/import/jobs", { params });
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function retryImportJob(jobId: number): Promise<ImportJobOut> {
|
||||||
|
const { data } = await api.post<ImportJobOut>(`/import/jobs/${jobId}/retry`);
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function cancelImportJob(jobId: number): Promise<ImportJobOut> {
|
||||||
|
const { data } = await api.post<ImportJobOut>(`/import/jobs/${jobId}/cancel`);
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
export default api;
|
export default api;
|
||||||
|
|||||||
@@ -1,92 +1,334 @@
|
|||||||
import { Button, Card, Table, Tabs, Upload, message } from "antd";
|
import { Button, Card, Progress, Select, Space, Table, Tag, Upload, message } from "antd";
|
||||||
import type { UploadProps } from "antd";
|
import type { UploadFile, UploadProps } from "antd";
|
||||||
import { useState } from "react";
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
import api from "../api";
|
import {
|
||||||
|
createImportJob,
|
||||||
|
getImportJob,
|
||||||
|
listImportJobs,
|
||||||
|
retryImportJob,
|
||||||
|
cancelImportJob,
|
||||||
|
type ImportJobOut
|
||||||
|
} from "../api";
|
||||||
|
|
||||||
|
type ImportMethod = "excel" | "ai";
|
||||||
|
|
||||||
|
const POLL_INTERVAL_MS = 1500;
|
||||||
|
const TERMINAL_STATUSES = ["success", "failed", "cancelled"];
|
||||||
|
const IMPORT_METHOD_KEY = "pb_import_method";
|
||||||
|
|
||||||
|
function isTerminal(status: string) {
|
||||||
|
return TERMINAL_STATUSES.includes(status);
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadStoredMethod(): ImportMethod {
|
||||||
|
try {
|
||||||
|
const v = sessionStorage.getItem(IMPORT_METHOD_KEY);
|
||||||
|
if (v === "excel" || v === "ai") return v;
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
return "ai";
|
||||||
|
}
|
||||||
|
|
||||||
export default function ImportPage() {
|
export default function ImportPage() {
|
||||||
const [preview, setPreview] = useState<Record<string, string>[]>([]);
|
const [method, setMethod] = useState<ImportMethod>(loadStoredMethod);
|
||||||
|
const [jobs, setJobs] = useState<ImportJobOut[]>([]);
|
||||||
|
const [pendingFiles, setPendingFiles] = useState<UploadFile[]>([]);
|
||||||
const [loading, setLoading] = useState(false);
|
const [loading, setLoading] = useState(false);
|
||||||
|
const pollTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||||
|
|
||||||
const aiProps: UploadProps = {
|
// Persist method so remounts (e.g. Strict Mode, route switch) restore selection
|
||||||
name: "file",
|
const setMethodAndStore = useCallback((value: ImportMethod) => {
|
||||||
customRequest: async ({ file, onSuccess, onError }) => {
|
setMethod(value);
|
||||||
|
try {
|
||||||
|
sessionStorage.setItem(IMPORT_METHOD_KEY, value);
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
const activeJobs = jobs.filter((j) => !isTerminal(j.status));
|
||||||
|
const hasActive = activeJobs.length > 0;
|
||||||
|
const pendingCount = pendingFiles.length;
|
||||||
|
|
||||||
|
const fetchJob = useCallback(async (jobId: number) => {
|
||||||
|
try {
|
||||||
|
const job = await getImportJob(jobId);
|
||||||
|
setJobs((prev) => {
|
||||||
|
const next = prev.map((j) => (j.id === jobId ? job : j));
|
||||||
|
if (!next.find((j) => j.id === jobId)) next.unshift(job);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
return job;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const startPolling = useCallback(() => {
|
||||||
|
if (pollTimerRef.current) return;
|
||||||
|
pollTimerRef.current = setInterval(() => {
|
||||||
|
setJobs((prev) => {
|
||||||
|
const toPoll = prev.filter((j) => !isTerminal(j.status));
|
||||||
|
toPoll.forEach((j) => {
|
||||||
|
getImportJob(j.id).then((job) => {
|
||||||
|
setJobs((p) => p.map((x) => (x.id === job.id ? job : x)));
|
||||||
|
}).catch(() => {});
|
||||||
|
});
|
||||||
|
return prev;
|
||||||
|
});
|
||||||
|
}, POLL_INTERVAL_MS);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const stopPolling = useCallback(() => {
|
||||||
|
if (pollTimerRef.current) {
|
||||||
|
clearInterval(pollTimerRef.current);
|
||||||
|
pollTimerRef.current = null;
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!hasActive) {
|
||||||
|
stopPolling();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
startPolling();
|
||||||
|
return () => { stopPolling(); };
|
||||||
|
}, [hasActive, startPolling, stopPolling]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
(async () => {
|
||||||
try {
|
try {
|
||||||
setLoading(true);
|
const list = await listImportJobs("queued,running");
|
||||||
const formData = new FormData();
|
if (!cancelled) {
|
||||||
formData.append("file", file as File);
|
setJobs((prev) => {
|
||||||
const { data } = await api.post("/import/ai/parse", formData);
|
const byId = new Map(prev.map((j) => [j.id, j]));
|
||||||
setPreview(data.preview || []);
|
list.forEach((j) => byId.set(j.id, j));
|
||||||
message.success("AI 解析完成");
|
return Array.from(byId.values()).sort((a, b) => b.id - a.id);
|
||||||
onSuccess?.({});
|
});
|
||||||
} catch (err) {
|
}
|
||||||
onError?.(err as Error);
|
} catch {
|
||||||
} finally {
|
// ignore
|
||||||
setLoading(false);
|
|
||||||
}
|
}
|
||||||
|
})();
|
||||||
|
return () => { cancelled = true; };
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const uploadProps: UploadProps = {
|
||||||
|
multiple: true,
|
||||||
|
beforeUpload: () => false,
|
||||||
|
fileList: pendingFiles,
|
||||||
|
onChange: (info) => {
|
||||||
|
setPendingFiles(info.fileList.slice(-100));
|
||||||
|
},
|
||||||
|
disabled: loading
|
||||||
|
};
|
||||||
|
|
||||||
|
const startImport = async () => {
|
||||||
|
if (!pendingFiles.length) {
|
||||||
|
message.info("请先添加文档");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const files = pendingFiles.map((f) => f.originFileObj).filter(Boolean) as File[];
|
||||||
|
if (!files.length) {
|
||||||
|
message.info("没有可上传的文件");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setLoading(true);
|
||||||
|
try {
|
||||||
|
const job = await createImportJob(files, method);
|
||||||
|
setJobs((prev) => [job, ...prev]);
|
||||||
|
setPendingFiles([]);
|
||||||
|
message.success("任务已入队,将按顺序处理");
|
||||||
|
startPolling();
|
||||||
|
} catch (err: unknown) {
|
||||||
|
const detail = err && typeof err === "object" && "response" in err
|
||||||
|
? (err as { response?: { data?: { detail?: string } } }).response?.data?.detail
|
||||||
|
: "创建任务失败";
|
||||||
|
message.error(String(detail));
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const excelProps: UploadProps = {
|
const handleRetry = async (jobId: number) => {
|
||||||
name: "file",
|
if (loading) return;
|
||||||
customRequest: async ({ file, onSuccess, onError }) => {
|
setLoading(true);
|
||||||
try {
|
try {
|
||||||
const formData = new FormData();
|
const newJob = await retryImportJob(jobId);
|
||||||
formData.append("file", file as File);
|
setJobs((prev) => [newJob, ...prev]);
|
||||||
const { data } = await api.post("/import/excel", formData);
|
message.success("已创建重试任务");
|
||||||
message.success(`Excel 导入成功,共 ${data.length} 题`);
|
startPolling();
|
||||||
onSuccess?.({});
|
} catch (err: unknown) {
|
||||||
} catch (err) {
|
const detail = err && typeof err === "object" && "response" in err
|
||||||
onError?.(err as Error);
|
? (err as { response?: { data?: { detail?: string } } }).response?.data?.detail
|
||||||
}
|
: "重试失败";
|
||||||
|
message.error(String(detail));
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const confirmSave = async () => {
|
const handleCancel = async (jobId: number) => {
|
||||||
await api.post("/import/ai/confirm", preview);
|
if (loading) return;
|
||||||
message.success(`已保存 ${preview.length} 道题`);
|
setLoading(true);
|
||||||
setPreview([]);
|
try {
|
||||||
|
const job = await cancelImportJob(jobId);
|
||||||
|
setJobs((prev) => prev.map((j) => (j.id === jobId ? job : j)));
|
||||||
|
message.success("已取消任务");
|
||||||
|
} catch {
|
||||||
|
message.error("取消失败");
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const clearFinished = () => {
|
||||||
|
setJobs((prev) => prev.filter((j) => !isTerminal(j.status)));
|
||||||
|
};
|
||||||
|
|
||||||
|
const progressPercent = (job: ImportJobOut) =>
|
||||||
|
job.total ? Number(((job.processed / job.total) * 100).toFixed(2)) : 0;
|
||||||
|
|
||||||
|
const statusTag = (status: string) => {
|
||||||
|
if (status === "queued") return <Tag>排队中</Tag>;
|
||||||
|
if (status === "running") return <Tag color="processing">处理中</Tag>;
|
||||||
|
if (status === "success") return <Tag color="success">成功</Tag>;
|
||||||
|
if (status === "failed") return <Tag color="error">失败</Tag>;
|
||||||
|
if (status === "cancelled") return <Tag>已取消</Tag>;
|
||||||
|
if (status === "retrying") return <Tag color="processing">重试中</Tag>;
|
||||||
|
return <Tag>{status}</Tag>;
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Tabs
|
<Card title="导入中心">
|
||||||
items={[
|
<Space direction="vertical" size={16} style={{ width: "100%" }}>
|
||||||
{
|
<Space wrap>
|
||||||
key: "ai",
|
<span>导入方式:</span>
|
||||||
label: "AI 智能导入",
|
<Select
|
||||||
children: (
|
value={method}
|
||||||
<Card>
|
onChange={(v) => {
|
||||||
<Upload {...aiProps} showUploadList={false}>
|
setMethodAndStore(v as ImportMethod);
|
||||||
<Button type="primary" loading={loading}>上传 PDF/Word 解析</Button>
|
}}
|
||||||
</Upload>
|
options={[
|
||||||
|
{ value: "excel", label: "Excel 批量导入" },
|
||||||
|
{ value: "ai", label: "AI 文档解析导入" }
|
||||||
|
]}
|
||||||
|
style={{ width: 220 }}
|
||||||
|
disabled={loading}
|
||||||
|
/>
|
||||||
|
<Upload {...uploadProps} showUploadList={{ showRemoveIcon: true }}>
|
||||||
|
<Button>添加文档</Button>
|
||||||
|
</Upload>
|
||||||
|
<Button
|
||||||
|
type="primary"
|
||||||
|
onClick={startImport}
|
||||||
|
loading={loading}
|
||||||
|
disabled={!pendingCount}
|
||||||
|
>
|
||||||
|
开始导入({pendingCount})
|
||||||
|
</Button>
|
||||||
|
<Button onClick={clearFinished} disabled={loading}>
|
||||||
|
清理已完成
|
||||||
|
</Button>
|
||||||
|
</Space>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
进行中:{activeJobs.length} | 成功/失败/已取消 已折叠,仅保留未完成任务
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<Table
|
||||||
|
rowKey="id"
|
||||||
|
dataSource={jobs}
|
||||||
|
pagination={{ pageSize: 10 }}
|
||||||
|
loading={loading}
|
||||||
|
expandable={{
|
||||||
|
expandedRowRender: (record) => (
|
||||||
<Table
|
<Table
|
||||||
style={{ marginTop: 16 }}
|
size="small"
|
||||||
rowKey={(_, index) => String(index)}
|
rowKey="id"
|
||||||
dataSource={preview}
|
dataSource={record.items || []}
|
||||||
columns={[
|
columns={[
|
||||||
{ title: "题干", dataIndex: "stem" },
|
{ title: "序号", dataIndex: "seq", width: 60 },
|
||||||
{ title: "答案", dataIndex: "answer", width: 120 },
|
{ title: "文件名", dataIndex: "filename" },
|
||||||
{ title: "题型", dataIndex: "question_type", width: 120 }
|
{
|
||||||
|
title: "状态",
|
||||||
|
dataIndex: "status",
|
||||||
|
width: 90,
|
||||||
|
render: (s: string) => statusTag(s)
|
||||||
|
},
|
||||||
|
{ title: "题目数", dataIndex: "question_count", width: 80 },
|
||||||
|
{ title: "错误", dataIndex: "error" }
|
||||||
]}
|
]}
|
||||||
pagination={{ pageSize: 10 }}
|
pagination={false}
|
||||||
/>
|
/>
|
||||||
<Button type="primary" onClick={confirmSave} disabled={!preview.length}>
|
)
|
||||||
确认保存
|
}}
|
||||||
</Button>
|
columns={[
|
||||||
</Card>
|
{ title: "任务 ID", dataIndex: "id", width: 80 },
|
||||||
)
|
{
|
||||||
},
|
title: "状态",
|
||||||
{
|
dataIndex: "status",
|
||||||
key: "excel",
|
width: 90,
|
||||||
label: "Excel 导入",
|
render: (s: string) => statusTag(s)
|
||||||
children: (
|
},
|
||||||
<Card>
|
{ title: "方式", dataIndex: "method", width: 80 },
|
||||||
<Upload {...excelProps} showUploadList={false}>
|
{
|
||||||
<Button type="primary">上传 Excel 导入</Button>
|
title: "进度",
|
||||||
</Upload>
|
key: "progress",
|
||||||
</Card>
|
width: 120,
|
||||||
)
|
render: (_: unknown, row: ImportJobOut) =>
|
||||||
}
|
`${row.processed}/${row.total}`
|
||||||
]}
|
},
|
||||||
/>
|
{
|
||||||
|
title: "进度条",
|
||||||
|
key: "progressBar",
|
||||||
|
width: 160,
|
||||||
|
render: (_: unknown, row: ImportJobOut) => (
|
||||||
|
<Progress
|
||||||
|
percent={progressPercent(row)}
|
||||||
|
size="small"
|
||||||
|
status={row.status === "failed" ? "exception" : undefined}
|
||||||
|
/>
|
||||||
|
)
|
||||||
|
},
|
||||||
|
{ title: "当前文件", dataIndex: "current_file", ellipsis: true },
|
||||||
|
{ title: "成功", dataIndex: "success_count", width: 70 },
|
||||||
|
{ title: "失败", dataIndex: "failed_count", width: 70 },
|
||||||
|
{ title: "错误", dataIndex: "error", ellipsis: true },
|
||||||
|
{
|
||||||
|
title: "操作",
|
||||||
|
width: 160,
|
||||||
|
render: (_: unknown, row: ImportJobOut) => (
|
||||||
|
<Space size="small">
|
||||||
|
{row.status === "failed" && row.failed_count > 0 && (
|
||||||
|
<Button
|
||||||
|
size="small"
|
||||||
|
disabled={loading}
|
||||||
|
onClick={() => handleRetry(row.id)}
|
||||||
|
>
|
||||||
|
重试失败项
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
{(row.status === "queued" || row.status === "running") && (
|
||||||
|
<Button
|
||||||
|
size="small"
|
||||||
|
danger
|
||||||
|
disabled={loading}
|
||||||
|
onClick={() => handleCancel(row.id)}
|
||||||
|
>
|
||||||
|
取消
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</Space>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
</Space>
|
||||||
|
</Card>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user