Skip to content

A
agentchat
Commit 38343fa4 authored by Wade's avatar Wade
Browse files
Options

update split doc swagger

parent 708cbf9f
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 308 additions and 356 deletions
docs/swagger.yaml
View file @ 38343fa4
...@@ -6,8 +6,8 @@ info: ...@@ -6,8 +6,8 @@ info:
paths: paths:
/split/document: /split/document:
post: post:
summary: Reconstruct document with Server-Sent Events summary: Reconstruct document
description: Reconstructs a document from a base64-encoded file for processing, supporting Server-Sent Events (SSE). description: Reconstructs a document from a base64-encoded file using the Tencent Cloud LKEAP API. Returns a JSON response containing the task result, including a URL to the reconstructed document when processing is complete.
tags: tags:
- Split - Split
requestBody: requestBody:
...@@ -15,46 +15,110 @@ paths: ...@@ -15,46 +15,110 @@ paths:
content: content:
application/json: application/json:
schema: schema:
type: object type: string
properties: description: The type of the file to be reconstructed (e.g., "TXT"). Currently supports "TXT" only.
FileType: example: "欢迎体验文档拆分功能。本测试文本旨在验证腾讯云 LKEAP API 的文档重建能力，确保文本按段落或句子正确拆分。"
type: string
description: The type of the file to be reconstructed
example: "TXT"
FileBase64:
type: string
description: The base64-encoded content of the file
example: "data:application/octet-stream;base64,4oCc6L+Z5bCx6LGh6Iqx5LiA5qC344CC5aaC5p6c5L2g54ix5LiK5LqG5LiA5py155Sf6ZW/IArlnKjkuIDpopfmmJ/mmJ/kuIrnmoToirHvvIzpgqPkuYjlpJzpl7TvvIwgCuS9oOeci+edgOWkqeepuuWwseaEn+WIsOeUnOicnOaEiSAK5b+r44CC5omA5pyJ55qE5pif5pif5LiK6YO9CuWlveixoeW8gOedgOiKseOAguKAnQ=="
required:
- FileType
- FileBase64
responses: responses:
'200': '200':
description: Successful reconstruction of the document with SSE description: Successful reconstruction of the document.
content: content:
text/event-stream: application/json:
schema: schema:
type: string type: object
description: Server-Sent Events stream containing the reconstructed document data properties:
result:
type: object
properties:
data:
type: array
items:
type: object
properties:
RequestId:
type: string
description: Unique request identifier.
example: "6654d909-71d4-418a-9687-412557b12ace"
TaskId:
type: string
description: Unique task identifier.
example: "6654d909-71d4-418a-9687-412557b12ace"
ResponseType:
type: string
description: Type of response (e.g., "TASK_RSP").
example: "TASK_RSP"
Progress:
type: string
description: Task progress percentage (e.g., "100" for complete).
example: "100"
ProgressMessage:
type: string
description: Human-readable progress message.
example: "完成文档解析"
DocumentRecognizeResultUrl:
type: string
description: URL to the reconstructed document (available when Progress is "100").
example: "https://lkeap-reconstruct-sse-prod-1251316161.cos.ap-guangzhou.myqcloud.com/output_files/atom_v3/2025-06-07/1257709057_100007669632/6654d909-71d4-418a-9687-412557b12ace_parse.zip?..."
StatusCode:
type: string
description: Status code of the task (e.g., "Success").
example: "Success"
required:
- RequestId
- TaskId
- ResponseType
- Progress
code:
type: integer
description: HTTP-like status code (200 for success).
example: 200
msg:
type: string
description: Additional message (empty for success).
example: ""
required:
- data
- code
- msg
required:
- result
examples: examples:
success: success:
value: | value:
data: {"status": "success", "document": "reconstructed content"} result:
event: message data:
id: 1 - RequestId: "6654d909-71d4-418a-9687-412557b12ace"
data: {"status": "complete"} TaskId: "6654d909-71d4-418a-9687-412557b12ace"
ResponseType: "TASK_RSP"
Progress: "100"
ProgressMessage: "完成文档解析"
DocumentRecognizeResultUrl: "https://lkeap-reconstruct-sse-prod-1251316161.cos.ap-guangzhou.myqcloud.com/output_files/atom_v3/2025-06-07/1257709057_100007669632/6654d909-71d4-418a-9687-412557b12ace_parse.zip?..."
StatusCode: "Success"
code: 200
msg: ""
'400': '400':
description: Invalid input or malformed request description: Invalid request parameters (e.g., missing FileType, invalid Base64, or file size exceeds 8MB).
content: content:
application/json: application/json:
schema: schema:
$ref: '#/components/schemas/Response' type: object
examples: properties:
error: error:
value: type: string
data: "" description: Error message.
code: 400 example:
msg: "Invalid file type or base64 content" error: "FileType is required"
'500':
description: Internal server error or API failure.
content:
application/json:
schema:
type: object
properties:
error:
type: string
description: Error message.
example:
error: "tencent cloud api error: InvalidParameter"
/index/document: /index/document:
post: post:
summary: Store Milvus index data summary: Store Milvus index data
......
flow.go
View file @ 38343fa4
...@@ -229,23 +229,13 @@ func DefineDocumentFlow(g *genkit.Genkit, indexer ai.Indexer) { ...@@ -229,23 +229,13 @@ func DefineDocumentFlow(g *genkit.Genkit, indexer ai.Indexer) {
}) })
} }
func DefineChatFlow(g *genkit.Genkit, retriever ai.Retriever, graphRetriever ai.Retriever, pgConnString string) { func DefineChatFlow(g *genkit.Genkit, retriever ai.Retriever, graphRetriever ai.Retriever, pgConnString string,kc *knowledge.KnowledgeClient ) {
qa, err := question.InitQAStore(pgConnString) qa, err := question.InitQAStore(pgConnString)
if err != nil { if err != nil {
log.Fatal().Msgf("InitQAStore failed: %v", err) log.Fatal().Msgf("InitQAStore failed: %v", err)
} }
// Initialize KnowledgeClient with test parameters
kc := knowledge.NewKnowledgeClient(knowledge.ClientConfig{
Endpoint: "lkeap.tencentcloudapi.com",
Region: "ap-guangzhou",
})
if err := kc.Init(context.Background()); err != nil {
log.Fatal().Msgf("Failed to initialize KnowledgeClient: %v", err)
}
log.Info().Msg("KnowledgeClient initialized successfully")
// Define a simple flow that generates jokes about a given topic // Define a simple flow that generates jokes about a given topic
genkit.DefineFlow(g, "chat", func(ctx context.Context, input *ChatInput) (Response, error) { genkit.DefineFlow(g, "chat", func(ctx context.Context, input *ChatInput) (Response, error) {
...@@ -444,3 +434,42 @@ func defineSimpleQaPrompt(g *genkit.Genkit, promptName string) (*ai.Prompt, erro ...@@ -444,3 +434,42 @@ func defineSimpleQaPrompt(g *genkit.Genkit, promptName string) (*ai.Prompt, erro
Msg("Prompt defined successfully") Msg("Prompt defined successfully")
return simpleQaPrompt, nil return simpleQaPrompt, nil
} }
func DefineSplitDocFlow(g *genkit.Genkit,kc *knowledge.KnowledgeClient ) {
// Define a simple flow that generates jokes about a given topic
genkit.DefineFlow(g, "split/document", func(ctx context.Context, input string) (SplitDocResponse, error) {
res,err := kc.ReconstructDocumentSSE(ctx,input)
if err != nil{
return SplitDocResponse{
Code: 500,
Msg: err.Error(),
},nil
}
for k,v := range res{
fmt.Println("process",k,*v.Progress)
}
return SplitDocResponse{
Code: 200,
Data: res,
},nil
})
}
type Response struct {
Data string `json:"data"`
Code int `json:"code"`
Msg string `json:"msg"`
}
type SplitDocResponse struct {
Data []knowledge.ReconstructDocumentSSEResponse `json:"data"`
Code int `json:"code"`
Msg string `json:"msg"`
}
main.go
View file @ 38343fa4
...@@ -10,6 +10,7 @@ import ( ...@@ -10,6 +10,7 @@ import (
"github.com/rs/zerolog" "github.com/rs/zerolog"
"github.com/wade-liwei/agentchat/plugins/deepseek" "github.com/wade-liwei/agentchat/plugins/deepseek"
"github.com/wade-liwei/agentchat/plugins/graphrag" // Import knowledge package "github.com/wade-liwei/agentchat/plugins/graphrag" // Import knowledge package
"github.com/wade-liwei/agentchat/plugins/knowledge"
"github.com/wade-liwei/agentchat/plugins/milvus" "github.com/wade-liwei/agentchat/plugins/milvus"
"github.com/firebase/genkit/go/plugins/evaluators" "github.com/firebase/genkit/go/plugins/evaluators"
...@@ -162,10 +163,24 @@ func main() { ...@@ -162,10 +163,24 @@ func main() {
log.Fatal().Msgf("graphrag.DefineIndexerAndRetriever failed: %v", err) log.Fatal().Msgf("graphrag.DefineIndexerAndRetriever failed: %v", err)
} }
DefineDocumentFlow(g, indexer) DefineDocumentFlow(g, indexer)
DefineGraphFlow(g, graphIndexer) DefineGraphFlow(g, graphIndexer)
DefineModelsFlow(g) DefineModelsFlow(g)
DefineChatFlow(g, retriever, graphRetriever, *pgConnString)
// Initialize KnowledgeClient with test parameters
kc := knowledge.NewKnowledgeClient(knowledge.ClientConfig{
Endpoint: "lkeap.tencentcloudapi.com",
Region: "ap-guangzhou",
})
if err := kc.Init(context.Background()); err != nil {
log.Fatal().Msgf("Failed to initialize KnowledgeClient: %v", err)
}
log.Info().Msg("KnowledgeClient initialized successfully")
DefineChatFlow(g, retriever, graphRetriever, *pgConnString,kc)
DefineSplitDocFlow(g,kc)
// 配置限速器：每秒 10 次请求，突发容量 20，最大并发 5 // 配置限速器：每秒 10 次请求，突发容量 20，最大并发 5
rl := NewRateLimiter(10, 20, 5) rl := NewRateLimiter(10, 20, 5)
...@@ -193,8 +208,3 @@ func main() { ...@@ -193,8 +208,3 @@ func main() {
} }
} }
type Response struct {
Data string `json:"data"`
Code int `json:"code"`
Msg string `json:"msg"`
}
plugins/knowledge/knowledge.go
View file @ 38343fa4
This diff is collapsed.
plugins/knowledge/knowledge_test.go
View file @ 38343fa4
...@@ -3,7 +3,6 @@ package knowledge ...@@ -3,7 +3,6 @@ package knowledge
import ( import (
"context" "context"
"os" "os"
"strings"
"testing" "testing"
"github.com/rs/zerolog" "github.com/rs/zerolog"
...@@ -116,97 +115,97 @@ func TestKnowledgeClient_QueryRewrite(t *testing.T) { ...@@ -116,97 +115,97 @@ func TestKnowledgeClient_QueryRewrite(t *testing.T) {
func TestKnowledgeClient_ReconstructDocumentSSE(t *testing.T) { // func TestKnowledgeClient_ReconstructDocumentSSE(t *testing.T) {
// Warning: Do not hardcode credentials in production code. Use environment variables or a secure vault. // // Warning: Do not hardcode credentials in production code. Use environment variables or a secure vault.
os.Setenv("TENCENTCLOUD_SECRET_ID", "AKID64oLfmfLtESUJ6i8LPSM4gCVbiniQuBF") // os.Setenv("TENCENTCLOUD_SECRET_ID", "AKID64oLfmfLtESUJ6i8LPSM4gCVbiniQuBF")
os.Setenv("TENCENTCLOUD_SECRET_KEY", "rX2JMBnBMJ2YqulOo37xa5OUMSN4Xnpd") // os.Setenv("TENCENTCLOUD_SECRET_KEY", "rX2JMBnBMJ2YqulOo37xa5OUMSN4Xnpd")
defer func() { // defer func() {
os.Unsetenv("TENCENTCLOUD_SECRET_ID") // os.Unsetenv("TENCENTCLOUD_SECRET_ID")
os.Unsetenv("TENCENTCLOUD_SECRET_KEY") // os.Unsetenv("TENCENTCLOUD_SECRET_KEY")
}() // }()
// Create client configuration // // Create client configuration
config := ClientConfig{ // config := ClientConfig{
Endpoint: "lkeap.tencentcloudapi.com", // Endpoint: "lkeap.tencentcloudapi.com",
Region: "ap-guangzhou", // Region: "ap-guangzhou",
} // }
// Initialize client // // Initialize client
client := NewKnowledgeClient(config) // client := NewKnowledgeClient(config)
ctx := context.Background() // ctx := context.Background()
// Test cases // // Test cases
tests := []struct { // tests := []struct {
name string // name string
text string // text string
initClient bool // initClient bool
expectError bool // expectError bool
}{ // }{
{ // {
name: "ValidText", // name: "ValidText",
text: "Sample text for reconstruction.\nAnother line.", // text: "Sample text for reconstruction.\nAnother line.",
initClient: true, // initClient: true,
expectError: true, // Expect error due to potentially invalid credentials // expectError: true, // Expect error due to potentially invalid credentials
}, // },
{ // {
name: "EmptyText", // name: "EmptyText",
text: "", // text: "",
initClient: true, // initClient: true,
expectError: true, // expectError: true,
}, // },
{ // {
name: "LargeText", // name: "LargeText",
text: strings.Repeat("a", 9*1024*1024), // ~9MB // text: strings.Repeat("a", 9*1024*1024), // ~9MB
initClient: true, // initClient: true,
expectError: true, // expectError: true,
}, // },
{ // {
name: "UninitializedClient", // name: "UninitializedClient",
text: "Sample text.", // text: "Sample text.",
initClient: false, // initClient: false,
expectError: true, // expectError: true,
}, // },
} // }
for _, tt := range tests { // for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { // t.Run(tt.name, func(t *testing.T) {
// Initialize client if required // // Initialize client if required
if tt.initClient { // if tt.initClient {
if err := client.Init(ctx); err != nil { // if err := client.Init(ctx); err != nil {
t.Fatalf("Failed to initialize KnowledgeClient: %v", err) // t.Fatalf("Failed to initialize KnowledgeClient: %v", err)
} // }
} // }
// Perform document reconstruction // // Perform document reconstruction
result, err := client.ReconstructDocumentSSE(ctx, tt.text) // result, err := client.ReconstructDocumentSSE(ctx, tt.text)
// Check error expectation // // Check error expectation
if tt.expectError { // if tt.expectError {
if err == nil { // if err == nil {
t.Error("Expected error, got none") // t.Error("Expected error, got none")
} else { // } else {
log.Debug(). // log.Debug().
Str("method", "TestKnowledgeClient_ReconstructDocumentSSE"). // Str("method", "TestKnowledgeClient_ReconstructDocumentSSE").
Str("test_name", tt.name). // Str("test_name", tt.name).
Str("error", err.Error()). // Str("error", err.Error()).
Msg("Received expected error") // Msg("Received expected error")
} // }
return // return
} // }
// Check response // // Check response
if err != nil { // if err != nil {
t.Errorf("ReconstructDocumentSSE failed: %v", err) // t.Errorf("ReconstructDocumentSSE failed: %v", err)
} // }
if result == "" { // if result == "" {
t.Error("Expected non-empty response JSON") // t.Error("Expected non-empty response JSON")
} // }
log.Info(). // log.Info().
Str("method", "TestKnowledgeClient_ReconstructDocumentSSE"). // Str("method", "TestKnowledgeClient_ReconstructDocumentSSE").
Str("test_name", tt.name). // Str("test_name", tt.name).
Str("response_json", result[:min(100, len(result))]). // Str("response_json", result[:min(100, len(result))]).
Msg("Document reconstruction successful") // Msg("Document reconstruction successful")
}) // })
} // }
} // }
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment