Commit 73b71380 authored by brent's avatar brent

modify S3

parent 124f16c4
No preview for this file type
File added
......@@ -14,10 +14,12 @@ imageUrl = "https://tmp-file.aigic.ai/api/v1/upload/persistence"
imageTransferUrl = "https://tmp-file.aigic.ai/api/v1/down/put"
replicateToken = "r8_9OCCea50go2Qkh0f0jhu3DbNjyzuyt61VNVI6"
replicateTimeout = 10
bucketName = "ai-watermark"
bucketName = "aon-watermark"
region = "us-west-2"
awsAccessKeyID = "AKIAYS2NSY7MVGYC2W7M"
awsSecretAccessKey = "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"
awsAccessKeyID = "ASIAU6GDZFLVTQUJAM5N"
awsSecretAccessKey = "rL9CGK+ljJ6BTJB48tlKZGv9fVb2rbVkRFqWqPQO"
awsSsessionToken = "IQoJb3JpZ2luX2VjENP//////////wEaCWV1LXdlc3QtMiJIMEYCIQD+okPPmKk2bGkQ7XJieB7eaVrLyMsBiKqDIKkSnBuGXwIhAMbqozgbnV54cDuwQ4UcM3NFVcchuXhC0n5cv19uxawwKvkCCMz//////////wEQABoMMzM5NzEzMDEwNDExIgyBhks7yqa6KqAET/kqzQLDC5JX6Jf7DJLHzqSALfEggPeS3opS9pljCWwjnHEfJJWTfhYpPxxu30LxQQd/ZIm/BBuX/q+YsrlD+XSBl4dPJfGs4jPZsGl6oNbeJL4N1y2tAP031uXxyxakKgwtCSn1G/cLPZQAYzxzvAO7zK5I0UFbhjYtP1ACqsUP6aFNmjwd9O7SaLqc0p6xYXQrqGBZzJzFm7t0J7isjxDQXeJrusoHZA7UCY9xbPe0VGeomGdd6cI3PZUIPqpN86x+qAfc6Vs3NC3SMUV3Vy2ZtFzCs6DdkyC1T0cgBChr+5dtgeGznXYBLlvJ9OQMJI9BVpeqCj7hDzzINxQykFVlhzz3RDDYmVOhrOt5VDjkgZLNfOYI6ZadVkU9n8XDqI/vS/0402xa5yq8ZLKsZspYQR2fvIUd6ljI7eXqijYwthYR90+Efz10oezExPRJY+4w6NH1tQY6pgFN+pRgiztiU3e9TWqrcdxSfmxttzKZ3HSbKdbTn7v5E2VVHbWPNT1hpsfvBC3kWCRcoNj/Ssx9zSybcK+RRjaVW/K6xJPqDFnUOdoII/QAPS1ZSQeekp+nS9qjnGgj4AiyZpPlzuTKabLrEXb0NrDUdNaj/m9KsWKezQ/Qu1uwb7LWC6XlYdaBdzC89SaLhcvB4/3DzK7OScxsxj1qNrnwN3rSX4pk"
watermarkUrl = "http://16.163.191.255:30008/add-watermark"
[test]
whoisApi = "aonet"
......@@ -27,10 +29,12 @@ imageUrl = "https://tmp-file.aigic.ai/api/v1/upload/persistence"
imageTransferUrl = "https://tmp-file.aigic.ai/api/v1/down/put"
replicateToken = "r8_9OCCea50go2Qkh0f0jhu3DbNjyzuyt61VNVI6"
replicateTimeout = 10
bucketName = "ai-watermark"
bucketName = "aon-watermark"
region = "us-west-2"
awsAccessKeyID = "AKIAYS2NSY7MVGYC2W7M"
awsSecretAccessKey = "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"
awsAccessKeyID = "ASIAU6GDZFLV7ZNJSXV7"
awsSecretAccessKey = "oR19/lkVDJmlscR6RLIz8bS4sRVZ1ZUbkICBZ6Wk"
watermarkUrl = "http://16.163.191.255:30008/add-watermark"
[prod]
whoisApi = "replicate"
......@@ -40,10 +44,12 @@ imageUrl = "https://tmp-file.aigic.ai/api/v1/upload/persistence"
imageTransferUrl = "https://tmp-file.aigic.ai/api/v1/down/put"
replicateToken = "r8_9OCCea50go2Qkh0f0jhu3DbNjyzuyt61VNVI6"
replicateTimeout = 10
bucketName = "ai-watermark"
bucketName = "aon-watermark"
region = "us-west-2"
awsAccessKeyID = "AKIAYS2NSY7MVGYC2W7M"
awsSecretAccessKey = "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"
awsAccessKeyID = "ASIAU6GDZFLV7ZNJSXV7"
awsSecretAccessKey = "oR19/lkVDJmlscR6RLIz8bS4sRVZ1ZUbkICBZ6Wk"
watermarkUrl = "http://16.163.191.255:30008/add-watermark"
......
......@@ -17,6 +17,8 @@ import (
"github.com/beego/beego/v2/core/logs"
beego "github.com/beego/beego/v2/server/web"
"github.com/fogleman/gg"
"github.com/h2non/filetype"
storage_go "github.com/supabase-community/storage-go"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"gopkg.in/yaml.v2"
......@@ -24,10 +26,12 @@ import (
"image/jpeg"
"image/png"
"io"
"mime/multipart"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"reflect"
"strconv"
"strings"
......@@ -200,14 +204,17 @@ func FindWatermarkFields(node models.JSONNode, prefix string, result *[]string)
}
}
func checkFileIsImage(files []string) bool {
func checkFileIsImage(files []string) int {
for _, value := range files {
_, ext := parseUrl(value)
if ext == "jpeg" || ext == ".jpeg" || ext == ".jpg" || ext == "jpg" || ext == "png" || ext == ".png" || ext == "webp" || ext == ".webp" {
return true
return 1
}
if ext == "mp4" || ext == ".mp4" {
return 2
}
}
return false
return 0
}
func findValueByPath(data interface{}, path string) (interface{}, bool) {
......@@ -281,6 +288,75 @@ func transferImagesToS3(images []string, task *models.Task) []string {
return images
}
func transferFileToS3(images []string, task *models.Task, needWatermark bool) []string {
watermarkURL := ""
apps, count, err := aonsupabase.MyClient.From("app").Select("", "exact", false).Eq("app_id", task.AppId).Execute()
if err == nil {
var temp []models.App
if err := json.Unmarshal(apps, &temp); err != nil {
logs.Debug("apps Unmarshal err = ", err)
}
if len(temp) > 0 {
app := temp[0]
logs.Debug("app = ", count, app)
var result []string
FindWatermarkFields(app.TemplateParams, "", &result)
for _, path := range result {
fmt.Println(path)
value, found := findValueByPath(app.ParamsValue, path)
if found {
if str, ok := value.(string); ok {
watermarkURL = str
}
fmt.Printf("Value at '%s': %v\n", path, value)
}
}
}
}
var backImages []string
for _, value := range images {
if needWatermark {
fileData, data, mime, _ := addWatermarkNew(value, watermarkURL)
if fileData != nil {
url, err := uploadToS3New(fileData, data, mime)
if err != nil {
logs.Debug("uploadToS3New err = ", err)
url, err = uploadToS3Supabase(fileData, data, mime)
}
if url != "" && err == nil {
backImages = append(backImages, url)
}
defer fileData.Close()
defer os.Remove(fileData.Name())
}
} else {
srcImg, err := downloadFile(value)
if err != nil {
logs.Debug("addWatermark downloadImage faild =", value)
continue
}
buffer := make([]byte, 1024)
_, err = srcImg.Read(buffer)
if err != nil {
fmt.Println("Error reading file:", err)
continue
}
contentType := http.DetectContentType(buffer)
url, _ := uploadToS3New(srcImg, buffer, contentType)
if url != "" {
backImages = append(backImages, url)
}
defer srcImg.Close()
defer os.Remove(srcImg.Name())
}
}
if len(backImages) > 0 {
return backImages
}
return images
}
func downloadImage(url string) (image.Image, string, error) {
resp, err := http.Get(url)
if err != nil {
......@@ -296,11 +372,73 @@ func downloadImage(url string) (image.Image, string, error) {
return img, format, nil
}
func downloadFile(url string) (*os.File, error) {
// 从 URL 下载文件
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// 创建临时文件
tmpFile, err := os.CreateTemp("", "downloaded-*"+filepath.Ext(url))
if err != nil {
return nil, err
}
// 将响应数据写入临时文件
_, err = io.Copy(tmpFile, resp.Body)
if err != nil {
return nil, err
}
// 重置文件偏移量
_, err = tmpFile.Seek(0, io.SeekStart)
if err != nil {
return nil, err
}
return tmpFile, nil
}
func ImageToFile(filename string, img image.Image, format string) (*os.File, []byte, error) {
// 创建文件
timestamp := time.Now().Unix()
timestampStr := strconv.FormatInt(timestamp, 10)
width := img.Bounds().Dx()
height := img.Bounds().Dy()
filename = timestampStr + "_" + "width=" + strconv.Itoa(width) + "&height=" + strconv.Itoa(height) + "_" + filename
file, err := os.Create(filename)
if err != nil {
return nil, nil, err
}
buf := new(bytes.Buffer)
if format == "jpeg" || format == ".jpeg" || format == ".jpg" || format == "jpg" {
err = jpeg.Encode(buf, img, nil)
} else if format == "png" || format == ".png" {
err = png.Encode(buf, img)
} else {
file.Close()
return nil, nil, err
}
_, err = file.Write(buf.Bytes())
if err != nil {
fmt.Println("Error writing to file:", err)
return nil, nil, err
}
// 返回文件指针
return file, buf.Bytes(), nil
}
func uploadToS3(bucket, key string, img image.Image, format string) (string, error) {
awsAccessKeyID, _ := beego.AppConfig.String("awsAccessKeyID")
awsSecretAccessKey, _ := beego.AppConfig.String("awsSecretAccessKey")
awsSsessionToken, _ := beego.AppConfig.String("awsSsessionToken")
region, _ := beego.AppConfig.String("region")
creds := credentials.NewStaticCredentialsProvider(awsAccessKeyID, awsSecretAccessKey, "")
creds := credentials.NewStaticCredentialsProvider(awsAccessKeyID, awsSecretAccessKey, awsSsessionToken)
cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region), config.WithCredentialsProvider(creds))
//sess, err := session.NewSession(&aws.Config{
// Region: region,
......@@ -352,6 +490,88 @@ func uploadToS3(bucket, key string, img image.Image, format string) (string, err
//return presignResult.URL, err
}
func uploadToS3New(file *os.File, data []byte, mime string) (string, error) {
awsAccessKeyID, _ := beego.AppConfig.String("awsAccessKeyID")
awsSecretAccessKey, _ := beego.AppConfig.String("awsSecretAccessKey")
awsSsessionToken, _ := beego.AppConfig.String("awsSsessionToken")
bucketName, _ := beego.AppConfig.String("bucketName")
region, _ := beego.AppConfig.String("region")
creds := credentials.NewStaticCredentialsProvider(awsAccessKeyID, awsSecretAccessKey, awsSsessionToken)
cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region), config.WithCredentialsProvider(creds))
//sess, err := session.NewSession(&aws.Config{
// Region: region,
// Credentials: credentials.NewStaticCredentials(awsAccessKeyID, awsSecretAccessKey, ""),
//})
if err != nil {
return "", err
}
fileInfo, err := file.Stat()
if err != nil {
return "", err
}
client := s3.NewFromConfig(cfg)
filePath := "watermark"
key := filePath + "/" + fileInfo.Name()
//size := fileInfo.Size()
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: bytes.NewReader(data),
ContentType: aws.String(mime),
ACL: types.ObjectCannedACLPublicRead, // 可根据需要更改
})
returnUrl := "https://" + bucketName + ".s3.amazonaws.com/" + key
if err != nil {
logs.Debug("S3 ", err)
return "", err
}
return returnUrl, err
//presignClient := s3.NewPresignClient(client)
//presignResult, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{
// Bucket: &bucket,
// Key: &key,
//})
//if err != nil {
// logs.Debug("failed to presign request, %v", err)
// return "", err
//}
//return presignResult.URL, err
}
func uploadToS3Supabase(file *os.File, data []byte, mime string) (string, error) {
fileInfo, err := file.Stat()
if err != nil {
return "", err
}
kind, err := filetype.Match(data)
if err != nil {
logs.Debug("filetype.Match error = ", err)
return "", err
}
temo := "no-cache"
mime = kind.MIME.Value
upsert := false
options := storage_go.FileOptions{
ContentType: &mime,
CacheControl: &temo,
Upsert: &upsert,
}
key := "watermark/" + fileInfo.Name()
_, err = aonsupabase.MyClient.Storage.UploadFile("prediction_result", key, bytes.NewReader(data), options)
if err != nil {
return "", err
}
url := aonsupabase.MyClient.Storage.GetPublicUrl("prediction_result", key)
return url.SignedURL, err
}
func addWatermark(sourceURL string, watermarkURL string) (string, error) {
bucketName, _ := beego.AppConfig.String("bucketName")
outputKey, ext := parseUrl(sourceURL)
......@@ -398,12 +618,142 @@ func addWatermark(sourceURL string, watermarkURL string) (string, error) {
s3Url, err := uploadToS3(bucketName, outputKey, outputImg, ext)
if err != nil {
logs.Debug("uploadToS3 faild", err)
return "", err
file, data, err := ImageToFile(outputKey, outputImg, ext)
if err != nil {
return "", err
}
s3Url, err = uploadToS3Supabase(file, data, "")
if err != nil {
return "", err
}
defer file.Close()
defer os.Remove(file.Name())
}
return s3Url, nil
}
func addWatermarkNew(sourceURL string, watermarkURL string) (*os.File, []byte, string, error) {
// 下载源图
srcImg, err := downloadFile(sourceURL)
if err != nil {
logs.Debug("addWatermarkNew downloadImage faild =", sourceURL)
return nil, nil, "", err
}
// 下载水印图
var watermarkImg *os.File
if watermarkURL == "" {
watermarkImg, err = os.Open("./watermark.png")
if err != nil {
logs.Debug("read watermark faild")
}
defer watermarkImg.Close()
} else {
watermarkImg, err = downloadFile(watermarkURL)
if err != nil {
logs.Debug("addWatermarkNew watermark faild", err)
}
}
host, _ := beego.AppConfig.String("watermarkUrl")
payload := &bytes.Buffer{}
writer := multipart.NewWriter(payload)
partA, err := writer.CreateFormFile("input", filepath.Base(srcImg.Name()))
if err != nil {
fmt.Println("Error creating form file A:", err)
return nil, nil, "", err
}
_, err = io.Copy(partA, srcImg)
if err != nil {
fmt.Println("Error copying file A:", err)
return nil, nil, "", err
}
partB, err := writer.CreateFormFile("watermark", filepath.Base(watermarkImg.Name()))
if err != nil {
fmt.Println("Error creating form file B:", err)
return nil, nil, "", err
}
_, err = io.Copy(partB, watermarkImg)
if err != nil {
fmt.Println("Error copying file B:", err)
return nil, nil, "", err
}
err = writer.Close()
if err != nil {
fmt.Println("Error closing writer:", err)
return nil, nil, "", err
}
client := &http.Client{}
request, err := http.NewRequest("POST", host, payload)
if err != nil {
logs.Info("addWatermarkNew Error NewRequest request:", err)
return nil, nil, "", err
}
request.Header.Set("Content-Type", writer.FormDataContentType())
logs.Info("addWatermarkNew client sending request:")
resp, err := client.Do(request)
if err != nil {
logs.Info("addWatermarkNew Error sending request:", err)
return nil, nil, "", err
}
defer resp.Body.Close()
logs.Info("addWatermarkNew resp code", resp.StatusCode)
body, err := io.ReadAll(resp.Body)
//logs.Info("addWatermarkNew body", string(body))
if resp.StatusCode == 200 && body == nil {
logs.Info("addWatermarkNew Body reading response:", err)
return nil, nil, "", err
}
if err != nil {
logs.Info("addWatermarkNew Error reading response:", err)
return nil, nil, "", err
}
// 创建临时文件
kind, err := filetype.Match(body)
if err != nil {
logs.Debug("filetype.Match error = ", err)
return nil, nil, "", err
}
timestamp := time.Now().Unix()
timestampStr := strconv.FormatInt(timestamp, 10)
file, err := os.Create("./" + timestampStr + "_watermarked_file." + kind.Extension)
if err != nil {
fmt.Println("Error creating file:", err)
return nil, nil, "", err
}
//defer file.Close()
_, err = file.Write(body)
if err != nil {
fmt.Println("Error writing to file:", err)
return nil, nil, "", err
}
//tmpFile, err := os.CreateTemp("", timestampStr+"_watermarked-*."+kind.Extension)
//if err != nil {
// return nil, nil, "", err
//}
//
//// 将响应数据写入临时文件
//_, err = io.Copy(tmpFile, resp.Body)
//if err != nil {
// return nil, nil, "", err
//}
//
//// 重置文件偏移量
//_, err = tmpFile.Seek(0, io.SeekStart)
//if err != nil {
// return nil, nil, "", err
//}
return file, body, kind.MIME.Value, nil
}
func parseUrl(urlStr string) (string, string) {
// 解析 URL
u, err := url.Parse(urlStr)
......@@ -884,8 +1234,10 @@ func doGetReplicate(url string, task *models.Task, taskResponse *models.TaskResp
task.ReplicateOutput = temp.Output
if len(output) > 0 {
isImage := checkFileIsImage(output)
if isImage {
if isImage == 1 {
task.Output = transferImagesToS3(output, task)
} else if isImage == 2 {
task.Output = transferFileToS3(output, task, true)
} else {
task.Output = transferImages(output)
}
......@@ -989,7 +1341,7 @@ func doGetReplicateNew(url string, task *models.Task, taskResponse *models.TaskR
task.Status = 2
task.ReplicateOutput = temp.Output
isImage := checkFileIsImage(output)
if isImage {
if isImage == 1 {
task.Output = transferImagesToS3(output, task)
} else {
task.Output = transferImages(output)
......
......@@ -39,6 +39,7 @@ require (
github.com/golang/snappy v0.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect
github.com/h2non/filetype v1.1.3 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/klauspost/compress v1.13.6 // indirect
......
......@@ -122,6 +122,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
......
{"/Users/brent/Documents/wubanWork/aon_app_server/controllers":1723333924719856567}
\ No newline at end of file
{"/Users/brent/Documents/wubanWork/aon_app_server/controllers":1723693546535841393}
\ No newline at end of file
......@@ -5,8 +5,8 @@ import (
"github.com/supabase-community/supabase-go"
)
var API_URL = "http://43.198.54.207:8000"
var API_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE"
var API_URL = "https://vbxtvjffhsirnyxjcuku.supabase.co"
var API_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InZieHR2amZmaHNpcm55eGpjdWt1Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3MjI1MTY1NjQsImV4cCI6MjAzODA5MjU2NH0.E6SFFojKzIZCRIQAhoJ49aO4yY-m1Mb2sZtSkVh-tBg"
var MyClient *supabase.Client
func init() {
......
watermark.png

4.08 KB | W: | H:

watermark.png

9.85 KB | W: | H:

watermark.png
watermark.png
watermark.png
watermark.png
  • 2-up
  • Swipe
  • Onion skin
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment