From 02583bd49fc57841bdcb05b02486e27dd868cf00 Mon Sep 17 00:00:00 2001
From: erda-bot <81558540+erda-bot@users.noreply.github.com>
Date: Tue, 16 Nov 2021 10:10:24 +0800
Subject: [PATCH] support spark crd use pyFiles (#3056) (#3058)

Co-authored-by: chengjoey <30427474+chengjoey@users.noreply.github.com>
---
 apistructs/bigdata.go                                        | 5 +++++
 .../plugins/scheduler/executor/plugins/k8sspark/k8sspark.go  | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/apistructs/bigdata.go b/apistructs/bigdata.go
index cc86ca05a70..7c44ddfde32 100644
--- a/apistructs/bigdata.go
+++ b/apistructs/bigdata.go
@@ -64,4 +64,9 @@ type SparkConf struct {
 	PythonVersion    *string         `json:"pythonVersion,omitempty"`
 	DriverResource   BigdataResource `json:"driverResource"`
 	ExecutorResource BigdataResource `json:"executorResource"`
+	Deps             Dependencies    `json:"deps"`
+}
+
+type Dependencies struct {
+	PyFiles []string `json:"pyFiles"`
 }
diff --git a/modules/pipeline/pipengine/actionexecutor/plugins/scheduler/executor/plugins/k8sspark/k8sspark.go b/modules/pipeline/pipengine/actionexecutor/plugins/scheduler/executor/plugins/k8sspark/k8sspark.go
index b674a70a434..ef31dfe044c 100644
--- a/modules/pipeline/pipengine/actionexecutor/plugins/scheduler/executor/plugins/k8sspark/k8sspark.go
+++ b/modules/pipeline/pipengine/actionexecutor/plugins/scheduler/executor/plugins/k8sspark/k8sspark.go
@@ -388,6 +388,11 @@ func (k *K8sSpark) generateKubeSparkJob(job *apistructs.JobFromUser, conf *apist
 		},
 	}
 
+	// add deps pyFiles
+	if len(conf.Spec.SparkConf.Deps.PyFiles) > 0 {
+		sparkApp.Spec.Deps.PyFiles = conf.Spec.SparkConf.Deps.PyFiles
+	}
+
 	if sparkApp.Spec.Type == sparkv1beta2.PythonApplicationType {
 		sparkApp.Spec.PythonVersion = stringptr("3")
 		if conf.Spec.SparkConf.PythonVersion != nil {