fix: get system datasource oss and embedding suite from pkgconfig

Signed-off-by: bjwswang <[email protected]>

apiserver/pkg/application/application.go

@@ -45,7 +45,6 @@ import (
 	"github.com/kubeagi/arcadia/apiserver/pkg/gpt"
 	"github.com/kubeagi/arcadia/apiserver/pkg/utils"
 	"github.com/kubeagi/arcadia/pkg/config"
-	"github.com/kubeagi/arcadia/pkg/datasource"
 	pkgutils "github.com/kubeagi/arcadia/pkg/utils"
 )
 
@@ -990,19 +989,11 @@ func UploadIcon(ctx context.Context, client client.Client, icon, appName, namesp
 			return "", err
 		}
 
-		system, err := config.GetSystemDatasource(ctx)
+		ds, err := config.GetSystemDatasourceOSS(ctx)
 		if err != nil {
 			return "", err
 		}
 
-		endpoint := system.Spec.Endpoint.DeepCopy()
-		if endpoint != nil && endpoint.AuthSecret != nil {
-			endpoint.AuthSecret.WithNameSpace(namespace)
-		}
-		ds, err := datasource.NewOSS(ctx, client, endpoint)
-		if err != nil {
-			return "", err
-		}
 		iconName := fmt.Sprintf("application/%s/icon/%s", appName, appName)
 		_, err = ds.Client.PutObject(ctx, namespace, iconName, bytes.NewReader(imgBytes), int64(len(imgBytes)), minio.PutObjectOptions{})
 		return icon, err

apiserver/pkg/chat/chat_docs.go

@@ -38,8 +38,7 @@ import (
 	"github.com/kubeagi/arcadia/apiserver/pkg/auth"
 	"github.com/kubeagi/arcadia/apiserver/pkg/chat/storage"
 	pkgclient "github.com/kubeagi/arcadia/apiserver/pkg/client"
-	"github.com/kubeagi/arcadia/apiserver/pkg/common"
-	"github.com/kubeagi/arcadia/pkg/config"
+	pkgconfig "github.com/kubeagi/arcadia/pkg/config"
 )
 
 // ReceiveConversationDocs receive and process docs for a conversation
@@ -80,7 +79,7 @@ func (cs *ChatServer) ReceiveConversationFile(ctx context.Context, messageID str
 	}
 
 	// upload files to system datasource
-	ds, err := common.SystemDatasourceOSS(ctx, cs.cli)
+	ds, err := pkgconfig.GetSystemDatasourceOSS(ctx)
 	if err != nil {
 		klog.Errorf("no storage service found with err %s", err)
 		return nil, fmt.Errorf("no storage service found with err %s", err.Error())
@@ -174,7 +173,7 @@ func (cs *ChatServer) ReceiveConversationFile(ctx context.Context, messageID str
 // Knoweledgebase will embed the document into vectorstore which can be used in this conversation as references(similarity search)
 func (cs *ChatServer) BuildConversationKnowledgeBase(ctx context.Context, req ConversationFilesReqBody, document storage.Document) error {
 	// get system embedding suite
-	embedder, vs, err := common.SystemEmbeddingSuite(ctx, cs.cli)
+	embedder, vs, err := pkgconfig.GetSystemEmbeddingSuite(ctx)
 	if err != nil {
 		return err
 	}
@@ -206,7 +205,7 @@ func (cs *ChatServer) BuildConversationKnowledgeBase(ctx context.Context, req Co
 		return err
 	}
 	// systemDatasource which stores the document
-	systemDatasource, err := config.GetSystemDatasource(ctx)
+	systemDatasource, err := pkgconfig.GetSystemDatasource(ctx)
 	if err != nil {
 		return err
 	}

apiserver/pkg/common/common.go

@@ -25,15 +25,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kubeagi/arcadia/api/base/v1alpha1"
 	"github.com/kubeagi/arcadia/apiserver/graph/generated"
 	"github.com/kubeagi/arcadia/apiserver/pkg/auth"
 	"github.com/kubeagi/arcadia/pkg/config"
-	"github.com/kubeagi/arcadia/pkg/datasource"
 )
 
 var (
@@ -59,42 +57,6 @@ var (
 	ModelSourceHuggingface = "huggingface"
 )
 
-func SystemDatasourceOSS(ctx context.Context, mgrClient client.Client) (*datasource.OSS, error) {
-	systemDatasource, err := config.GetSystemDatasource(ctx)
-	if err != nil {
-		return nil, err
-	}
-	endpoint := systemDatasource.Spec.Endpoint.DeepCopy()
-	if endpoint.AuthSecret != nil && endpoint.AuthSecret.Namespace == nil {
-		endpoint.AuthSecret.WithNameSpace(systemDatasource.Namespace)
-	}
-	return datasource.NewOSS(ctx, mgrClient, endpoint)
-}
-
-// SystemEmbeddingSuite returns the embedder and vectorstore which are built-in in system config
-// Embedder and vectorstore are both required when generating a new embedding.That's why we call it a `EmbeddingSuit`
-func SystemEmbeddingSuite(ctx context.Context, cli client.Client) (*v1alpha1.Embedder, *v1alpha1.VectorStore, error) {
-	// get the built-in system embedder
-	emd, err := config.GetEmbedder(ctx)
-	if err != nil {
-		return nil, nil, err
-	}
-	embedder := &v1alpha1.Embedder{}
-	if err := cli.Get(ctx, types.NamespacedName{Namespace: *emd.Namespace, Name: emd.Name}, embedder); err != nil {
-		return nil, nil, err
-	}
-	// get the built-in system vectorstore
-	vs, err := config.GetVectorStore(ctx)
-	if err != nil {
-		return nil, nil, err
-	}
-	vectorStore := &v1alpha1.VectorStore{}
-	if err := cli.Get(ctx, types.NamespacedName{Namespace: *vs.Namespace, Name: vs.Name}, vectorStore); err != nil {
-		return nil, nil, err
-	}
-	return embedder, vectorStore, nil
-}
-
 // GetAPIServer returns the api server url to access arcadia's worker
 // if external is true,then this func will return the external api server
 func GetAPIServer(ctx context.Context, cli client.Client, external bool) (string, error) {

apiserver/pkg/knowledgebase/knowledgebase.go

@@ -31,10 +31,9 @@ import (
 
 	"github.com/kubeagi/arcadia/api/base/v1alpha1"
 	"github.com/kubeagi/arcadia/apiserver/graph/generated"
-	pkgclient "github.com/kubeagi/arcadia/apiserver/pkg/client"
 	"github.com/kubeagi/arcadia/apiserver/pkg/common"
 	graphqlutils "github.com/kubeagi/arcadia/apiserver/pkg/utils"
-	"github.com/kubeagi/arcadia/pkg/config"
+	pkgconfig "github.com/kubeagi/arcadia/pkg/config"
 )
 
 func knowledgebase2modelConverter(ctx context.Context, c client.Client) func(obj client.Object) (generated.PageNode, error) {
@@ -100,8 +99,10 @@ func knowledgebase2model(ctx context.Context, c client.Client, knowledgebase *v1
 		})
 	}
 	if len(knowledgebase.Status.FileGroupDetail) > 0 {
-		systemClient, _ := pkgclient.GetClient(nil)
-		oss, _ := common.SystemDatasourceOSS(ctx, systemClient)
+		oss, err := pkgconfig.GetSystemDatasourceOSS(ctx)
+		if err != nil {
+			return nil, err
+		}
 
 		for _, filegroupdetail := range knowledgebase.Status.FileGroupDetail {
 			ns := knowledgebase.Namespace
@@ -203,7 +204,7 @@ func knowledgebase2model(ctx context.Context, c client.Client, knowledgebase *v1
 func CreateKnowledgeBase(ctx context.Context, c client.Client, input generated.CreateKnowledgeBaseInput) (*generated.KnowledgeBase, error) {
 	var filegroups []v1alpha1.FileGroup
 	var vectorstore v1alpha1.TypedObjectReference
-	vector, _ := config.GetVectorStore(ctx)
+	vector, _ := pkgconfig.GetVectorStore(ctx)
 	displayname, description, embedder := "", "", ""
 	if input.DisplayName != nil {
 		displayname = *input.DisplayName

apiserver/pkg/model/model.go

@@ -33,9 +33,9 @@ import (
 	"github.com/kubeagi/arcadia/api/base/v1alpha1"
 	"github.com/kubeagi/arcadia/apiserver/config"
 	"github.com/kubeagi/arcadia/apiserver/graph/generated"
-	pkgclient "github.com/kubeagi/arcadia/apiserver/pkg/client"
 	"github.com/kubeagi/arcadia/apiserver/pkg/common"
 	graphqlutils "github.com/kubeagi/arcadia/apiserver/pkg/utils"
+	pkgconfig "github.com/kubeagi/arcadia/pkg/config"
 	"github.com/kubeagi/arcadia/pkg/utils"
 )
 
@@ -247,18 +247,14 @@ func ReadModel(ctx context.Context, c client.Client, name, namespace string) (*g
 	return obj2model(u)
 }
 
-func ModelFiles(ctx context.Context, c client.Client, modelName, namespace string, input *generated.FileFilter) (*generated.PaginatedResult, error) {
+func ModelFiles(ctx context.Context, modelName, namespace string, input *generated.FileFilter) (*generated.PaginatedResult, error) {
 	prefix := fmt.Sprintf("model/%s/", modelName)
 	keyword := ""
 	if input != nil && input.Keyword != nil {
 		keyword = *input.Keyword
 	}
 
-	systemClient, err := pkgclient.GetClient(nil)
-	if err != nil {
-		return nil, err
-	}
-	oss, err := common.SystemDatasourceOSS(ctx, systemClient)
+	oss, err := pkgconfig.GetSystemDatasourceOSS(ctx)
 	if err != nil {
 		return nil, err
 	}

apiserver/pkg/rag/rag.go

@@ -36,6 +36,7 @@ import (
 	"github.com/kubeagi/arcadia/apiserver/pkg/auth"
 	"github.com/kubeagi/arcadia/apiserver/pkg/common"
 	graphqlutils "github.com/kubeagi/arcadia/apiserver/pkg/utils"
+	pkgconfig "github.com/kubeagi/arcadia/pkg/config"
 	"github.com/kubeagi/arcadia/pkg/utils"
 )
 
@@ -517,7 +518,7 @@ func GetV1alpha1RAG(ctx context.Context, kubeClient client.Client, name, namespa
 }
 
 func getFiles(ctx context.Context, kubeClient client.Client, bucket string, files []string) ([]*generated.F, error) {
-	oss, err := common.SystemDatasourceOSS(ctx, kubeClient)
+	oss, err := pkgconfig.GetSystemDatasourceOSS(ctx)
 	if err != nil {
 		return nil, err
 	}

apiserver/pkg/rag/report.go

@@ -27,10 +27,10 @@ import (
 
 	"github.com/minio/minio-go/v7"
 	"k8s.io/klog/v2"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kubeagi/arcadia/api/evaluation/v1alpha1"
 	"github.com/kubeagi/arcadia/apiserver/pkg/common"
+	pkgconfig "github.com/kubeagi/arcadia/pkg/config"
 )
 
 const (
@@ -135,11 +135,11 @@ type (
 )
 
 func ParseSummary(
-	ctx context.Context, c client.Client,
+	ctx context.Context,
 	appName, ragName, namespace string,
 	metricThresholds map[string]float64,
 ) (Report, error) {
-	source, err := common.SystemDatasourceOSS(ctx, c)
+	source, err := pkgconfig.GetSystemDatasourceOSS(ctx)
 	if err != nil {
 		klog.Errorf("failed to get system datasource error %s", err)
 		return Report{}, err
@@ -212,8 +212,8 @@ func ParseSummary(
 	return report, nil
 }
 
-func PraseScatterChart(ctx context.Context, c client.Client, appName, ragName, namespace string) ([]ScatterData, error) {
-	source, err := common.SystemDatasourceOSS(ctx, c)
+func PraseScatterChart(ctx context.Context, appName, ragName, namespace string) ([]ScatterData, error) {
+	source, err := pkgconfig.GetSystemDatasourceOSS(ctx)
 	if err != nil {
 		klog.Errorf("failed to get system datasource error %s", err)
 		return nil, err
@@ -271,11 +271,11 @@ func PraseScatterChart(ctx context.Context, c client.Client, appName, ragName, n
 }
 
 func ParseResult(
-	ctx context.Context, c client.Client,
+	ctx context.Context,
 	page, pageSize int,
 	appName, ragName, namespace, sortBy, order string,
 ) (ReportDetail, error) {
-	source, err := common.SystemDatasourceOSS(ctx, c)
+	source, err := pkgconfig.GetSystemDatasourceOSS(ctx)
 	if err != nil {
 		klog.Errorf("failed to get system datasource error %s", err)
 		return ReportDetail{}, err

apiserver/pkg/versioneddataset/versioned_dataset.go

@@ -32,9 +32,9 @@ import (
 
 	"github.com/kubeagi/arcadia/api/base/v1alpha1"
 	"github.com/kubeagi/arcadia/apiserver/graph/generated"
-	pkgclient "github.com/kubeagi/arcadia/apiserver/pkg/client"
 	"github.com/kubeagi/arcadia/apiserver/pkg/common"
 	graphqlutils "github.com/kubeagi/arcadia/apiserver/pkg/utils"
+	pkgconfig "github.com/kubeagi/arcadia/pkg/config"
 	"github.com/kubeagi/arcadia/pkg/utils"
 )
 
@@ -104,11 +104,7 @@ func VersionFiles(ctx context.Context, _ client.Client, input *generated.Version
 		keyword = *filter.Keyword
 	}
 
-	systemClient, err := pkgclient.GetClient(nil)
-	if err != nil {
-		return nil, err
-	}
-	oss, err := common.SystemDatasourceOSS(ctx, systemClient)
+	oss, err := pkgconfig.GetSystemDatasourceOSS(ctx)
 	if err != nil {
 		return nil, err
 	}