refactor(scheduler): 优化爬取和简历处理流程,改为异步线程池执行
- 将手动触发爬取任务改为使用FastAPI后台任务执行 - 在职位处理逻辑中,将获取候选人列表改为线程池异步调用,避免阻塞事件循环 - 在候选人处理流程中,将获取简历详情改为线程池异步调用 - 在入库操作中使用线程池异步执行,提升处理性能 - 在Boss爬取任务中,将获取职位列表和获取候选人操作改为线程池异步调用 - 统一改造调用同步爬虫方法为异步线程池调用,提升整体异步性能和响应速度
This commit is contained in:
@@ -16,7 +16,7 @@ if str(src_path) not in sys.path:
|
||||
sys.path.insert(0, str(src_path))
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, BackgroundTasks
|
||||
|
||||
from cn.yinlihupo.ylhp_hr_2_0.controller.api import create_app
|
||||
from cn.yinlihupo.ylhp_hr_2_0.service.scheduler import get_scheduler
|
||||
@@ -61,10 +61,10 @@ def create_combined_app(enable_scheduler: bool = True) -> FastAPI:
|
||||
return {"success": True, "message": f"Job {job_id} resumed"}
|
||||
|
||||
@app.post("/api/scheduler/trigger/crawl")
|
||||
async def trigger_crawl():
|
||||
async def trigger_crawl(background_tasks: BackgroundTasks):
|
||||
"""手动触发爬取任务"""
|
||||
scheduler = get_scheduler()
|
||||
asyncio.create_task(scheduler._crawl_boss())
|
||||
background_tasks.add_task(scheduler._crawl_boss)
|
||||
return {"success": True, "message": "Crawl task triggered"}
|
||||
|
||||
return app
|
||||
|
||||
Reference in New Issue
Block a user