|
|
@@ -1,19 +1,34 @@
|
|
|
package com.storlead.knowledge.api;
|
|
|
|
|
|
+import com.alibaba.fastjson.JSONArray;
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
|
import com.fasterxml.jackson.core.type.TypeReference;
|
|
|
+import com.fasterxml.jackson.databind.JsonMappingException;
|
|
|
import com.storlead.framework.common.result.Result;
|
|
|
import com.storlead.knowledge.config.DifyProperties;
|
|
|
+import com.storlead.knowledge.entity.AiDatasetsFileEntity;
|
|
|
+import com.storlead.knowledge.entity.AiTrainingEntity;
|
|
|
import com.storlead.knowledge.pojo.dto.KnowledgeDTO;
|
|
|
import com.storlead.knowledge.utils.HttpService;
|
|
|
import com.storlead.knowledge.utils.JacksonHolder;
|
|
|
import com.storlead.knowledge.pojo.dto.QueryPageDTO;
|
|
|
+import com.storlead.knowledge.pojo.dto.AiTrainingDTO;
|
|
|
+import com.storlead.knowledge.pojo.dto.AiDatasetsFileDTO;
|
|
|
+import com.storlead.knowledge.pojo.vo.AiTrainingVO;
|
|
|
+import com.storlead.knowledge.pojo.vo.AiDatasetsFileVO;
|
|
|
+import com.storlead.knowledge.service.AiTrainingService;
|
|
|
+import com.storlead.knowledge.service.AiDatasetsFileService;
|
|
|
+import com.baomidou.mybatisplus.core.metadata.IPage;
|
|
|
import io.swagger.annotations.Api;
|
|
|
import io.swagger.annotations.ApiOperation;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.web.bind.annotation.*;
|
|
|
+import org.springframework.web.multipart.MultipartFile;
|
|
|
|
|
|
import javax.annotation.Resource;
|
|
|
+import javax.validation.Valid;
|
|
|
+import java.util.List;
|
|
|
import java.util.Map;
|
|
|
|
|
|
|
|
|
@@ -25,6 +40,10 @@ public class KnowledgeController {
|
|
|
private HttpService httpService;
|
|
|
@Resource
|
|
|
private DifyProperties difyProperties;
|
|
|
+ @Resource
|
|
|
+ private AiTrainingService aiTrainingService;
|
|
|
+ @Resource
|
|
|
+ private AiDatasetsFileService aiDatasetsFileService;
|
|
|
|
|
|
@GetMapping("datasets")
|
|
|
@ApiOperation("获取所有知识库数据集")
|
|
|
@@ -90,4 +109,182 @@ public class KnowledgeController {
|
|
|
String url = difyProperties.getBaseUrl() + "workspaces/current/models/model-types/"+model_type;
|
|
|
return httpService.get(url, null, "Bearer "+difyProperties.getDatasetApiKey(), new TypeReference<>() {});
|
|
|
}
|
|
|
+
|
|
|
+ // ==================== AI训练记录管理 ====================
|
|
|
+
|
|
|
+ @GetMapping("training/page")
|
|
|
+ @ApiOperation("分页获取AI训练记录")
|
|
|
+ public Result<IPage<AiTrainingEntity>> pageTraining(@RequestParam(defaultValue = "1") Integer pageIndex,
|
|
|
+ @RequestParam(defaultValue = "10") Integer pageSize,
|
|
|
+ @RequestParam(required = false) String keyword,
|
|
|
+ @RequestParam(required = false) String datasetsId) {
|
|
|
+ IPage<AiTrainingEntity> result = aiTrainingService.pageVO(pageIndex, pageSize, keyword, datasetsId);
|
|
|
+ //https://api.dify.ai/v1/datasets/{dataset_id}/documents/{batch}/indexing-status
|
|
|
+ String url = difyProperties.getBaseUrl() ;
|
|
|
+ for (AiTrainingEntity vo : result.getRecords()) {
|
|
|
+ if (vo.getStatus()==0){
|
|
|
+ List<AiDatasetsFileEntity> files = aiDatasetsFileService.pageVO(1, 1000, null, vo.getId()).getRecords();
|
|
|
+ Boolean flag = true;
|
|
|
+ for (AiDatasetsFileEntity file : files) {
|
|
|
+ String url2 = difyProperties.getBaseUrl() + "datasets/"+vo.getDatasetsId()+"/documents/"+file.getBatch()+"/indexing-status";
|
|
|
+ Result<Object> result1 = httpService.get(url2, null, "Bearer "+difyProperties.getDatasetApiKey(), new TypeReference<>() {});
|
|
|
+ if (result1.isSuccess()) {
|
|
|
+ // 从Dify返回结果中提取document_id
|
|
|
+ Object responseData = result1.getResult();
|
|
|
+
|
|
|
+ JSONObject root = new JSONObject((Map<String, Object>) responseData);
|
|
|
+ JSONArray data = root.getJSONArray("data");
|
|
|
+ String indexing_status = data.getJSONObject(0).getString("indexing_status");
|
|
|
+
|
|
|
+// String indexing_status = null;
|
|
|
+// if (responseData instanceof Map) {
|
|
|
+// Object data = ((Map<?, ?>) responseData).get("data");
|
|
|
+// if (data instanceof Map) {
|
|
|
+// Object status = ((Map<?, ?>) data).get("indexing_status");
|
|
|
+// if (status != null) {
|
|
|
+// indexing_status = status.toString();
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+ file.setIndexingStatus(indexing_status);
|
|
|
+ aiDatasetsFileService.updateById(file);
|
|
|
+ if (!"completed".equals(indexing_status)) {
|
|
|
+ flag = false;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (flag) {
|
|
|
+ vo.setStatus(1);
|
|
|
+ aiTrainingService.updateById(vo);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return Result.ok(result);
|
|
|
+ }
|
|
|
+
|
|
|
+ @GetMapping("training/{id}")
|
|
|
+ @ApiOperation("根据ID获取AI训练记录")
|
|
|
+ public Result<AiTrainingEntity> getTrainingById(@PathVariable Long id) {
|
|
|
+ AiTrainingEntity vo = aiTrainingService.getById(id);
|
|
|
+ return Result.ok(vo);
|
|
|
+ }
|
|
|
+
|
|
|
+ @PostMapping("training")
|
|
|
+ @ApiOperation("创建AI训练记录(上传文件并导入知识库)")
|
|
|
+ /***
|
|
|
+ * {"datasetsId":"xxx", "name":"训练标题", "description":"描述"}
|
|
|
+ */
|
|
|
+ public Result<Object> createTraining(
|
|
|
+ @RequestParam("file") MultipartFile[] files,
|
|
|
+ @RequestParam("data") String data
|
|
|
+ ) throws JsonProcessingException {
|
|
|
+
|
|
|
+ // 1. 解析DTO
|
|
|
+ AiTrainingEntity entity = JacksonHolder.OBJECT_MAPPER.readValue(data, AiTrainingEntity.class);
|
|
|
+
|
|
|
+ // 2. 保存训练记录到数据库
|
|
|
+ boolean saved = aiTrainingService.save(entity);
|
|
|
+ if (!saved) {
|
|
|
+ return Result.error("保存训练记录失败");
|
|
|
+ }
|
|
|
+
|
|
|
+ // 3. 逐个上传文件到知识库
|
|
|
+ String url = difyProperties.getBaseUrl() + "datasets/" + entity.getDatasetsId() + "/document/create-by-file";
|
|
|
+ for (MultipartFile file : files) {
|
|
|
+ if (file.isEmpty()) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ Result<Object> uploadResult = httpService.uploadWithJson(
|
|
|
+ url,
|
|
|
+ file,
|
|
|
+ data,
|
|
|
+ "Bearer " + difyProperties.getDatasetApiKey(),
|
|
|
+ new TypeReference<>() {
|
|
|
+ }
|
|
|
+ );
|
|
|
+
|
|
|
+ // 4. 如果上传成功,保存文件记录到数据库
|
|
|
+ if (uploadResult.isSuccess()) {
|
|
|
+ // 从Dify返回结果中提取document_id
|
|
|
+ Object responseData = uploadResult.getResult();
|
|
|
+ String documentId = null;
|
|
|
+ String batch = null;
|
|
|
+ String indexing_status = null;
|
|
|
+ if (responseData instanceof Map) {
|
|
|
+ Object document = ((Map<?, ?>) responseData).get("document");
|
|
|
+ if (document instanceof Map) {
|
|
|
+ Object id = ((Map<?, ?>) document).get("id");
|
|
|
+ if (id != null) {
|
|
|
+ documentId = id.toString();
|
|
|
+ }
|
|
|
+ Object indexing_status_ob = ((Map<?, ?>) document).get("indexing_status");
|
|
|
+ if (id != null) {
|
|
|
+ indexing_status = indexing_status_ob.toString();
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ batch = ((Map<?, ?>) responseData).get("batch").toString();
|
|
|
+ }
|
|
|
+
|
|
|
+ AiDatasetsFileEntity fileEntity = new AiDatasetsFileEntity();
|
|
|
+ fileEntity.setFileName(file.getOriginalFilename());
|
|
|
+ fileEntity.setDocumentId(documentId);
|
|
|
+ fileEntity.setAiTrainingId(entity.getId());
|
|
|
+ fileEntity.setIndexingStatus(indexing_status);
|
|
|
+ fileEntity.setBatch(batch);
|
|
|
+ aiDatasetsFileService.save(fileEntity);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return Result.ok("上传成功");
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ @DeleteMapping("training/{id}")
|
|
|
+ @ApiOperation("删除AI训练记录")
|
|
|
+ public Result<Boolean> deleteTraining(@PathVariable Long id) {
|
|
|
+ boolean result = aiTrainingService.removeById(id);
|
|
|
+ return Result.ok(result);
|
|
|
+ }
|
|
|
+
|
|
|
+ // ==================== AI训练文件管理 ====================
|
|
|
+
|
|
|
+ @GetMapping("training/file/page")
|
|
|
+ @ApiOperation("分页获取AI训练文件")
|
|
|
+ public Result<IPage<AiDatasetsFileEntity>> pageTrainingFile(@RequestParam(defaultValue = "1") Integer pageIndex,
|
|
|
+ @RequestParam(defaultValue = "10") Integer pageSize,
|
|
|
+ @RequestParam(required = false) String keyword,
|
|
|
+ @RequestParam(required = false) Long aiTrainingId) {
|
|
|
+ IPage<AiDatasetsFileEntity> result = aiDatasetsFileService.pageVO(pageIndex, pageSize, keyword, aiTrainingId);
|
|
|
+ return Result.ok(result);
|
|
|
+ }
|
|
|
+
|
|
|
+// @GetMapping("training/file/{id}")
|
|
|
+// @ApiOperation("根据ID获取AI训练文件")
|
|
|
+// public Result<AiDatasetsFileEntity> getTrainingFileById(@PathVariable Long id) {
|
|
|
+// AiDatasetsFileEntity vo = aiDatasetsFileService.getById(id);
|
|
|
+// return Result.ok(vo);
|
|
|
+// }
|
|
|
+
|
|
|
+// @PostMapping("training/file")
|
|
|
+// @ApiOperation("创建AI训练文件")
|
|
|
+// public Result<Boolean> createTrainingFile(@Valid @RequestBody AiDatasetsFileEntity dto) {
|
|
|
+// boolean result = aiDatasetsFileService.save(dto);
|
|
|
+// return Result.ok(result);
|
|
|
+// }
|
|
|
+
|
|
|
+// @PutMapping("training/file")
|
|
|
+// @ApiOperation("更新AI训练文件")
|
|
|
+// public Result<Boolean> updateTrainingFile(@Valid @RequestBody AiDatasetsFileEntity dto) {
|
|
|
+// boolean result = aiDatasetsFileService.updateById(dto);
|
|
|
+// return Result.ok(result);
|
|
|
+// }
|
|
|
+
|
|
|
+// @DeleteMapping("training/file/{id}")
|
|
|
+// @ApiOperation("删除AI训练文件")
|
|
|
+// public Result<Boolean> deleteTrainingFile(@PathVariable Long id) {
|
|
|
+// boolean result = aiDatasetsFileService.removeById(id);
|
|
|
+// return Result.ok(result);
|
|
|
+// }
|
|
|
}
|