Merge remote-tracking branch 'origin/master'

java/storlead-knowledge/storlead-knowledge-api/src/main/java/com/storlead/knowledge/api/ChunkController.java

@@ -6,6 +6,7 @@ import com.storlead.framework.common.result.Result;
 import com.storlead.knowledge.config.DifyProperties;
 import com.storlead.knowledge.pojo.dto.ChunkDTO;
 import com.storlead.knowledge.pojo.dto.QueryPageDTO;
+import com.storlead.knowledge.pojo.vo.KnowledgeStatisticsVO;
 import com.storlead.knowledge.utils.HttpService;
 import com.storlead.knowledge.utils.JacksonHolder;
 import io.swagger.annotations.Api;
@@ -13,6 +14,9 @@ import io.swagger.annotations.ApiOperation;
 import org.springframework.web.bind.annotation.*;
 
 import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 
@@ -32,7 +36,7 @@ public class ChunkController {
         String url = difyProperties.getBaseUrl()  + "datasets/"+ dataset_id +"/documents/"+document_id +"/segments";
         return httpService.get(
                 url,
-                page.getKeyword()==null?Map.of("page", page.getPageIndex(), "size", page.getPageSize()):Map.of("page", page.getPageIndex(), "size", page.getPageSize(), "keyword", page.getKeyword()),
+                page.getKeyword()==null?Map.of("page", page.getPageIndex(), "limit", page.getPageSize()):Map.of("page", page.getPageIndex(), "limit", page.getPageSize(), "keyword", page.getKeyword()),
                 "Bearer "+difyProperties.getDatasetApiKey(),
                 new TypeReference<>() {}
         );
@@ -93,8 +97,98 @@ public class ChunkController {
                 JacksonHolder.OBJECT_MAPPER.writeValueAsString(request),
                 new TypeReference<>() {}
         );
+    }
+
+    @GetMapping("getAllChunks")
+    @ApiOperation("获取所有的文本块")
+    public Result<Object> getAllChunks(QueryPageDTO page) {
+        // 1. 调用Dify API获取所有知识库列表（使用较大分页以获取全部数据）
+        String url = difyProperties.getBaseUrl() + "datasets";
+        Result<Object> datasetsResult = httpService.get(
+                url,
+                Map.of("page", 1, "limit", 1000),
+                "Bearer " + difyProperties.getDatasetApiKey(),
+                new TypeReference<>() {}
+        );
 
+        List<Object> results =  new ArrayList<>();
+        if (datasetsResult.isSuccess()) {
+            Object responseData = datasetsResult.getResult();
+            if (responseData instanceof Map) {
+                Map<?, ?> responseMap = (Map<?, ?>) responseData;
+                // 遍历数据集，累加文档数和文本块数
+                Object dataObj = responseMap.get("data");
+                if (dataObj instanceof List<?> dataList) {
+                    for (Object item : dataList) {
+                        if (item instanceof Map<?, ?> datasetMap) {
+                            String datasets_id = datasetMap.get("id").toString();
+                            //获取知识库中文档的列表
+                            String url1 = difyProperties.getBaseUrl() + "datasets/"+datasets_id+"/documents";
+                            Result<Object> documents_list  = httpService.get(
+                                    url1,
+                                    Map.of("page", 1, "limit", 10000),
+                                    "Bearer " + difyProperties.getDatasetApiKey(),
+                                    new TypeReference<>() {}
+                            );
 
-    }
+                            if (documents_list.isSuccess()) {
+                                Object responseData1 = documents_list.getResult();
+                                Map<?, ?> responseMap2 = (Map<?, ?>) responseData1;
+                                Object dataObj1 = responseMap2.get("data");
+                                if (dataObj1 instanceof List<?> dataList1) {
+                                    for (Object item1 : dataList1) {
+                                        if (item1 instanceof Map<?, ?> datasetMap1) {
+                                            String document_id = datasetMap1.get("id").toString();
+                                            String url2 = difyProperties.getBaseUrl() + "datasets/" + datasets_id + "/documents/" + document_id+"/segments";
+                                            Result<Object> chunks_result = httpService.get(
+                                                    url2,
+                                                    page.getKeyword()==null?Map.of("page", 1, "limit", 10000):Map.of("page", 1, "limit", 10000,"keyword", page.getKeyword()),
+                                                    "Bearer " + difyProperties.getDatasetApiKey(),
+                                                    new TypeReference<>() {
+                                                    });
+                                            if (chunks_result.isSuccess()) {
+                                                Object responseData2 = chunks_result.getResult();
+                                                Map<?, ?> responseMap3 = (Map<?, ?>) responseData2;
+                                                Object dataObj2 = responseMap3.get("data");
+                                                if (dataObj2 instanceof List<?> dataList2) {
+                                                    //results.addAll(dataList2);
+                                                    for (Object item2 : dataList2) {
+                                                        @SuppressWarnings("unchecked")
+                                                        Map<String, Object> datasetMap2 = (Map<String, Object>) item2;
+                                                        datasetMap2.put("dataset_id", datasets_id);
+                                                        results.add(datasetMap2);
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
 
+        // 对结果进行分页
+        int total = results.size();
+        int pageIndex = page.getPageIndex() != null ? page.getPageIndex() : 1;
+        int pageSize = page.getPageSize() != null ? page.getPageSize() : 10;
+        int fromIndex = (pageIndex - 1) * pageSize;
+        int toIndex = Math.min(fromIndex + pageSize, total);
+
+        List<Object> pagedResults;
+        if (fromIndex >= total) {
+            pagedResults = new ArrayList<>();
+        } else {
+            pagedResults = results.subList(fromIndex, toIndex);
+        }
+        Map<String,Object> resultMap = new HashMap<>() ;
+        resultMap.put("data", pagedResults);
+        resultMap.put("total", total);
+        resultMap.put("limit", pageSize);
+        resultMap.put("page", pageIndex);
+        resultMap.put("has_more", total > (fromIndex + pageSize));
+        return Result.ok(resultMap);
+    }
 }

java/storlead-knowledge/storlead-knowledge-api/src/main/java/com/storlead/knowledge/api/DocumentController.java

@@ -36,7 +36,7 @@ public class DocumentController {
         String url = difyProperties.getBaseUrl() +"datasets/"+ dataset_id + "/documents";
         return httpService.get(
                 url,
-                page.getKeyword()==null?Map.of("page", page.getPageIndex(), "size", page.getPageSize()):Map.of("page", page.getPageIndex(), "size", page.getPageSize(), "keyword", page.getKeyword()),
+                page.getKeyword()==null?Map.of("page", page.getPageIndex(), "limit", page.getPageSize()):Map.of("page", page.getPageIndex(), "limit", page.getPageSize(), "keyword", page.getKeyword()),
                 "Bearer "+difyProperties.getDatasetApiKey(),
                 new TypeReference<>() {}
         );

java/storlead-knowledge/storlead-knowledge-api/src/main/java/com/storlead/knowledge/api/KnowledgeController.java

@@ -50,7 +50,7 @@ public class KnowledgeController {
         String url = difyProperties.getBaseUrl() + "datasets";
         return httpService.get(
                 url,
-                page.getKeyword()==null?Map.of("page", page.getPageIndex(), "size", page.getPageSize()):Map.of("page", page.getPageIndex(), "size", page.getPageSize(), "keyword", page.getKeyword()),
+                page.getKeyword()==null?Map.of("page", page.getPageIndex(), "limit", page.getPageSize()):Map.of("page", page.getPageIndex(), "limit", page.getPageSize(), "keyword", page.getKeyword()),
                 "Bearer "+difyProperties.getDatasetApiKey(),
                 new TypeReference<>() {}
         );
@@ -65,7 +65,7 @@ public class KnowledgeController {
         String url = difyProperties.getBaseUrl() + "datasets";
         Result<Object> datasetsResult = httpService.get(
                 url,
-                Map.of("page", 1, "size", 1000),
+                Map.of("page", 1, "limit", 1000),
                 "Bearer " + difyProperties.getDatasetApiKey(),
                 new TypeReference<>() {}
         );
@@ -103,7 +103,7 @@ public class KnowledgeController {
                             String url1 = difyProperties.getBaseUrl() + "datasets/"+datasets_id+"/documents";
                             Result<Object> documents_list  = httpService.get(
                                     url1,
-                                    Map.of("page", 1, "size", 1000),
+                                    Map.of("page", 1, "limit", 1000),
                                     "Bearer " + difyProperties.getDatasetApiKey(),
                                     new TypeReference<>() {}
                             );

ui/smarttrade-platform/src/views/KnowledgeView.vue

@@ -594,7 +594,7 @@ const newKnowledge = ref({
   text: '',
   name: '',
   indexing_technique: "high_quality",
-  doc_form: "text_model",
+  doc_form: "qa_model",
   doc_language:"Chinese Simplified"
 })