refactor(scheduler): 优化爬取和简历处理流程,改为异步线程池执行

- 将手动触发爬取任务改为使用FastAPI后台任务执行
- 在职位处理逻辑中,将获取候选人列表改为线程池异步调用,避免阻塞事件循环
- 在候选人处理流程中,将获取简历详情改为线程池异步调用
- 在入库操作中使用线程池异步执行,提升处理性能
- 在Boss爬取任务中,将获取职位列表和获取候选人操作改为线程池异步调用
- 统一改造调用同步爬虫方法为异步线程池调用,提升整体异步性能和响应速度
This commit is contained in:
2026-03-25 11:50:34 +08:00
parent fc24e3a37b
commit af11f8ad48
4 changed files with 51 additions and 45 deletions

View File

@@ -16,7 +16,7 @@ if str(src_path) not in sys.path:
sys.path.insert(0, str(src_path))
import uvicorn
from fastapi import FastAPI
from fastapi import FastAPI, BackgroundTasks
from cn.yinlihupo.ylhp_hr_2_0.controller.api import create_app
from cn.yinlihupo.ylhp_hr_2_0.service.scheduler import get_scheduler
@@ -61,10 +61,10 @@ def create_combined_app(enable_scheduler: bool = True) -> FastAPI:
return {"success": True, "message": f"Job {job_id} resumed"}
@app.post("/api/scheduler/trigger/crawl")
async def trigger_crawl():
async def trigger_crawl(background_tasks: BackgroundTasks):
"""手动触发爬取任务"""
scheduler = get_scheduler()
asyncio.create_task(scheduler._crawl_boss())
background_tasks.add_task(scheduler._crawl_boss)
return {"success": True, "message": "Crawl task triggered"}
return app