From 164aa5c9385a6616ef53e21aa06dbc4cc453696c Mon Sep 17 00:00:00 2001 From: Sonal Goyal Date: Wed, 16 Oct 2024 16:47:22 +0530 Subject: [PATCH] test exec utors compile but tdf validation fails --- .../common/core/executor/FindAndLabeller.java | 10 ++++++ .../common/core/executor/TrainMatcher.java | 18 ++++++++++ .../core/executor/ExecutorValidator.java | 6 ++-- .../core/executor/FindAndLabelValidator.java | 6 ++-- .../core/executor/LabellerValidator.java | 8 ++--- .../common/core/executor/LinkerValidator.java | 4 +-- .../core/executor/MatcherValidator.java | 6 ++-- .../core/executor/TestExecutorsCompound.java | 7 ++-- .../core/executor/TestExecutorsGeneric.java | 6 ++-- .../core/executor/TrainMatchValidator.java | 6 ++-- .../core/executor/TrainerValidator.java | 13 +++++-- .../executor/TrainingDataFinderValidator.java | 6 ++-- .../spark/core/context/ZinggSparkContext.java | 34 ------------------- .../core/executor/TestSparkExecutors.java | 13 +++++-- .../spark/core/executor/ZinggSparkTester.java | 1 + 15 files changed, 82 insertions(+), 62 deletions(-) diff --git a/common/core/src/main/java/zingg/common/core/executor/FindAndLabeller.java b/common/core/src/main/java/zingg/common/core/executor/FindAndLabeller.java index b57ea7360..3afe4de28 100644 --- a/common/core/src/main/java/zingg/common/core/executor/FindAndLabeller.java +++ b/common/core/src/main/java/zingg/common/core/executor/FindAndLabeller.java @@ -40,4 +40,14 @@ public Labeller getLabeller(){ return this.labeller; } + public TrainingDataFinder getFinder() { + return finder; + } + + public void setFinder(TrainingDataFinder finder) { + this.finder = finder; + } + + + } diff --git a/common/core/src/main/java/zingg/common/core/executor/TrainMatcher.java b/common/core/src/main/java/zingg/common/core/executor/TrainMatcher.java index b4fdfc97e..80231966a 100644 --- a/common/core/src/main/java/zingg/common/core/executor/TrainMatcher.java +++ b/common/core/src/main/java/zingg/common/core/executor/TrainMatcher.java @@ -33,5 +33,23 @@ public void execute() throws ZinggClientException { trainer.execute(); matcher.execute(); } + + public Trainer getTrainer() { + return trainer; + } + + public void setTrainer(Trainer trainer) { + this.trainer = trainer; + } + + public Matcher getMatcher() { + return matcher; + } + + public void setMatcher(Matcher matcher) { + this.matcher = matcher; + } + + } diff --git a/common/core/src/test/java/zingg/common/core/executor/ExecutorValidator.java b/common/core/src/test/java/zingg/common/core/executor/ExecutorValidator.java index f42a4ac2e..daa62021e 100644 --- a/common/core/src/test/java/zingg/common/core/executor/ExecutorValidator.java +++ b/common/core/src/test/java/zingg/common/core/executor/ExecutorValidator.java @@ -9,10 +9,10 @@ public abstract class ExecutorValidator { public static final Log LOG = LogFactory.getLog(ExecutorValidator.class); - public ZinggBase executorObj; + public ZinggBase executor; - public ExecutorValidator(ZinggBase executorObj) { - this.executorObj = executorObj; + public ExecutorValidator(ZinggBase executor) { + this.executor = executor; } public abstract void validateResults() throws ZinggClientException; diff --git a/common/core/src/test/java/zingg/common/core/executor/FindAndLabelValidator.java b/common/core/src/test/java/zingg/common/core/executor/FindAndLabelValidator.java index 027425fc4..e99bcb192 100644 --- a/common/core/src/test/java/zingg/common/core/executor/FindAndLabelValidator.java +++ b/common/core/src/test/java/zingg/common/core/executor/FindAndLabelValidator.java @@ -16,8 +16,10 @@ public class FindAndLabelValidator extends ExecutorValidator tdfv; public LabellerValidator lv; - public FindAndLabelValidator(FindAndLabeller validator) { - super(validator); + public FindAndLabelValidator(FindAndLabeller executor) { + super(executor); + this.tdfv = new TrainingDataFinderValidator(executor.getFinder()); + this.lv = new LabellerValidator (executor.getLabeller()); } @Override diff --git a/common/core/src/test/java/zingg/common/core/executor/LabellerValidator.java b/common/core/src/test/java/zingg/common/core/executor/LabellerValidator.java index 4d04f9ba9..a68fe9b54 100644 --- a/common/core/src/test/java/zingg/common/core/executor/LabellerValidator.java +++ b/common/core/src/test/java/zingg/common/core/executor/LabellerValidator.java @@ -13,15 +13,15 @@ public class LabellerValidator extends ExecutorValidator validator) { - super(validator); + public LabellerValidator(Labeller executor) { + super(executor); } @Override public void validateResults() throws ZinggClientException { // check that marked data has at least 1 match row and 1 unmatch row - ZFrame dfMarked = validator.getContext().getPipeUtil(). - read(false, false, validator.getContext().getPipeUtil().getTrainingDataMarkedPipe(validator.getArgs())); + ZFrame dfMarked = executor.getContext().getPipeUtil(). + read(false, false, executor.getContext().getPipeUtil().getTrainingDataMarkedPipe(executor.getArgs())); C matchCond = dfMarked.equalTo(ColName.MATCH_FLAG_COL, 1); C notMatchCond = dfMarked.equalTo(ColName.MATCH_FLAG_COL, 0); diff --git a/common/core/src/test/java/zingg/common/core/executor/LinkerValidator.java b/common/core/src/test/java/zingg/common/core/executor/LinkerValidator.java index 617150646..9f0ab01dc 100644 --- a/common/core/src/test/java/zingg/common/core/executor/LinkerValidator.java +++ b/common/core/src/test/java/zingg/common/core/executor/LinkerValidator.java @@ -13,8 +13,8 @@ public class LinkerValidator extends MatcherValidator validator) { - super(validator); + public LinkerValidator(Matcher executor) { + super(executor); } @Override diff --git a/common/core/src/test/java/zingg/common/core/executor/MatcherValidator.java b/common/core/src/test/java/zingg/common/core/executor/MatcherValidator.java index ff119476d..0d5a232d2 100644 --- a/common/core/src/test/java/zingg/common/core/executor/MatcherValidator.java +++ b/common/core/src/test/java/zingg/common/core/executor/MatcherValidator.java @@ -13,8 +13,8 @@ public class MatcherValidator extends ExecutorValidator validator) { - super(validator); + public MatcherValidator(Matcher executor) { + super(executor); } @Override @@ -70,7 +70,7 @@ protected void testAccuracy(ZFrame gold, ZFrame result) throws public ZFrame getOutputData() throws ZinggClientException { - ZFrame output = validator.getContext().getPipeUtil().read(false, false, validator.getArgs().getOutput()[0]); + ZFrame output = executor.getContext().getPipeUtil().read(false, false, executor.getArgs().getOutput()[0]); return output; } diff --git a/common/core/src/test/java/zingg/common/core/executor/TestExecutorsCompound.java b/common/core/src/test/java/zingg/common/core/executor/TestExecutorsCompound.java index 38fd6f393..be88f0ebb 100644 --- a/common/core/src/test/java/zingg/common/core/executor/TestExecutorsCompound.java +++ b/common/core/src/test/java/zingg/common/core/executor/TestExecutorsCompound.java @@ -19,8 +19,11 @@ public TestExecutorsCompound() { @Override public List> getExecutors() throws ZinggClientException{ - executorTesterList.add(new ExecutorTester(getFindAndLabeller(), new FindAndLabelValidator(getFindAndLabeller()))); - executorTesterList.add(new ExecutorTester(getFindAndLabeller(), new FindAndLabelValidator(getFindAndLabeller()))); + FindAndLabeller findAndLabel = getFindAndLabeller(); + FindAndLabelValidator falValidator = new FindAndLabelValidator(findAndLabel); + ExecutorTester et = new ExecutorTester(findAndLabel, falValidator); + executorTesterList.add(et); + executorTesterList.add(et); executorTesterList.add(new ExecutorTester(getTrainMatcher(),getTrainMatchValidator(getTrainMatcher()))); return executorTesterList; } diff --git a/common/core/src/test/java/zingg/common/core/executor/TestExecutorsGeneric.java b/common/core/src/test/java/zingg/common/core/executor/TestExecutorsGeneric.java index 55a7fc744..d429afb36 100644 --- a/common/core/src/test/java/zingg/common/core/executor/TestExecutorsGeneric.java +++ b/common/core/src/test/java/zingg/common/core/executor/TestExecutorsGeneric.java @@ -50,9 +50,11 @@ public List> getExecutors() throws ZinggClientExce executorTesterList.add(new ExecutorTester(getLabeller(), new LabellerValidator(getLabeller()))); executorTesterList.add(new ExecutorTester(getTrainingDataFinder(), new TrainingDataFinderValidator(getTrainingDataFinder()))); executorTesterList.add(new ExecutorTester(getLabeller(), new LabellerValidator(getLabeller()))); + executorTesterList.add(new ExecutorTester(getTrainingDataFinder(), new TrainingDataFinderValidator(getTrainingDataFinder()))); + executorTesterList.add(new ExecutorTester(getLabeller(), new LabellerValidator(getLabeller()))); executorTesterList.add(new ExecutorTester(getTrainer(),getTrainerValidator(getTrainer()))); - executorTesterList.add(new ExecutorTester(getMatcher(),new MatcherValidator(getMatcher()))); - executorTesterList.add(new ExecutorTester(getLinker(),new LinkerValidator(getLinker()))); + executorTesterList.add(new ExecutorTester(getMatcher(),new MatcherValidator(getMatcher()))); + executorTesterList.add(new ExecutorTester(getLinker(),new LinkerValidator(getLinker()))); return executorTesterList; } diff --git a/common/core/src/test/java/zingg/common/core/executor/TrainMatchValidator.java b/common/core/src/test/java/zingg/common/core/executor/TrainMatchValidator.java index 2fb8d67fa..3d506baf5 100644 --- a/common/core/src/test/java/zingg/common/core/executor/TrainMatchValidator.java +++ b/common/core/src/test/java/zingg/common/core/executor/TrainMatchValidator.java @@ -18,9 +18,11 @@ public class TrainMatchValidator extends ExecutorValidator tv; MatcherValidator mv; - public TrainMatchValidator(TrainMatcher validator, IArguments args) { - super(validator); + public TrainMatchValidator(TrainMatcher executor, IArguments args) { + super(executor); this.args = args; + tv = new TrainerValidator(executor.getTrainer(), args); + mv = new MatcherValidator(executor.getMatcher()); } @Override diff --git a/common/core/src/test/java/zingg/common/core/executor/TrainerValidator.java b/common/core/src/test/java/zingg/common/core/executor/TrainerValidator.java index f6caea2d4..011161990 100644 --- a/common/core/src/test/java/zingg/common/core/executor/TrainerValidator.java +++ b/common/core/src/test/java/zingg/common/core/executor/TrainerValidator.java @@ -4,16 +4,23 @@ import org.apache.commons.logging.LogFactory; import zingg.common.client.IArguments; +import zingg.common.client.ZinggClientException; -public abstract class TrainerValidator extends ExecutorValidator { +public class TrainerValidator extends ExecutorValidator { public static final Log LOG = LogFactory.getLog(TrainerValidator.class); protected IArguments args; - public TrainerValidator(Trainer validator,IArguments args) { - super(validator); + public TrainerValidator(Trainer executor,IArguments args) { + super(executor); this.args = args; } + @Override + public void validateResults() throws ZinggClientException { + //doesnt do anything + //TODO - add modele xistence checks + } + } diff --git a/common/core/src/test/java/zingg/common/core/executor/TrainingDataFinderValidator.java b/common/core/src/test/java/zingg/common/core/executor/TrainingDataFinderValidator.java index 3ef6c5f8d..98a303c43 100644 --- a/common/core/src/test/java/zingg/common/core/executor/TrainingDataFinderValidator.java +++ b/common/core/src/test/java/zingg/common/core/executor/TrainingDataFinderValidator.java @@ -12,14 +12,14 @@ public class TrainingDataFinderValidator extends ExecutorValidato public static final Log LOG = LogFactory.getLog(TrainingDataFinderValidator.class); - public TrainingDataFinderValidator(TrainingDataFinder validator) { - super(validator); + public TrainingDataFinderValidator(TrainingDataFinder executor) { + super(executor); } @Override public void validateResults() throws ZinggClientException { // check that unmarked data has at least 10 rows - ZFrame df = validator.getContext().getPipeUtil().read(false, false, validator.getContext().getPipeUtil().getTrainingDataUnmarkedPipe(validator.getArgs())); + ZFrame df = executor.getContext().getPipeUtil().read(false, false, executor.getContext().getPipeUtil().getTrainingDataUnmarkedPipe(executor.getArgs())); long trainingDataCount = df.count(); assertTrue(trainingDataCount > 10); diff --git a/spark/core/src/main/java/zingg/spark/core/context/ZinggSparkContext.java b/spark/core/src/main/java/zingg/spark/core/context/ZinggSparkContext.java index 07e57d652..ce42bd833 100644 --- a/spark/core/src/main/java/zingg/spark/core/context/ZinggSparkContext.java +++ b/spark/core/src/main/java/zingg/spark/core/context/ZinggSparkContext.java @@ -35,46 +35,12 @@ public class ZinggSparkContext extends Context, Row,C public static final Log LOG = LogFactory.getLog(ZinggSparkContext.class); -<<<<<<< HEAD - public void initSessionAndContext(SparkSession session) - throws ZinggClientException { - try{ -// if (session==null) { -// session = SparkSession -// .builder() -// .appName("Zingg") -// .getOrCreate(); -// -// //session = new SparkSession(spark, license); -// } - this.session = session; - if (ctx==null) { - ctx = JavaSparkContext.fromSparkContext(session.sparkContext()); - JavaSparkContext.jarOfClass(IZingg.class); - LOG.debug("Context " + ctx.toString()); - //initHashFns(); - ctx.setCheckpointDir("/tmp/checkpoint"); - } - } - catch(Throwable e) { - if (LOG.isDebugEnabled()) e.printStackTrace(); - throw new ZinggClientException(e.getMessage()); - } -======= @Override public void init(SparkSession session) throws ZinggClientException { this.session = session; setUtils(); ->>>>>>> 622a907d (init changes to spark) - } - - @Override - public void init(SparkSession session) - throws ZinggClientException { - initSessionAndContext(session); - setUtils(); } @Override diff --git a/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutors.java b/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutors.java index 98f975aee..72ba2ea48 100644 --- a/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutors.java +++ b/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutors.java @@ -5,6 +5,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -12,6 +13,7 @@ import org.apache.spark.sql.types.DataType; import org.junit.jupiter.api.AfterEach; +import zingg.common.client.IZingg; import zingg.common.client.ZinggClientException; import zingg.common.core.executor.Labeller; import zingg.common.core.executor.TestExecutorsCompound; @@ -21,11 +23,11 @@ import zingg.common.core.executor.TrainerValidator; import zingg.spark.core.context.ZinggSparkContext; -public class TestSparkExecutors extends TestExecutorsCompound,Row,Column,DataType> { +public class TestSparkExecutors extends TestExecutorsGeneric,Row,Column,DataType> { protected static final String CONFIG_FILE = "zingg/spark/core/executor/configSparkIntTest.json"; protected static final String TEST_DATA_FILE = "zingg/spark/core/executor/test.csv"; - protected static final String CONFIGLINK_FILE = "ingg/spark/core/executor/configSparkLinkTest.json"; + protected static final String CONFIGLINK_FILE = "zingg/spark/core/executor/configSparkLinkTest.json"; protected static final String TEST1_DATA_FILE = "zingg/spark/core/executor/test1.csv"; protected static final String TEST2_DATA_FILE = "zingg/spark/core/executor/test2.csv"; @@ -39,6 +41,11 @@ public TestSparkExecutors() throws IOException, ZinggClientException { .master("local[*]") .appName("Zingg" + "Junit") .getOrCreate(); + + JavaSparkContext ctx1 = new JavaSparkContext(spark.sparkContext()); + JavaSparkContext.jarOfClass(IZingg.class); + ctx1.setCheckpointDir("/tmp/checkpoint"); + this.ctx = new ZinggSparkContext(); this.ctx.setSession(spark); this.ctx.setUtils(); @@ -84,6 +91,7 @@ protected SparkLinker getLinker() throws ZinggClientException { protected SparkTrainerTester getTrainerValidator(Trainer,Row,Column,DataType> trainer) { return new SparkTrainerTester(trainer,args); } + /* @Override protected SparkFindAndLabeller getFindAndLabeller() throws ZinggClientException { @@ -102,6 +110,7 @@ protected SparkTrainMatcher getTrainMatcher() throws ZinggClientException { protected SparkTrainMatchTester getTrainMatchValidator(TrainMatcher,Row,Column,DataType> trainMatch) { return new SparkTrainMatchTester(trainMatch,args); } + */ @Override public String setupArgs() throws ZinggClientException, IOException { diff --git a/spark/core/src/test/java/zingg/spark/core/executor/ZinggSparkTester.java b/spark/core/src/test/java/zingg/spark/core/executor/ZinggSparkTester.java index 447ac780d..bf7ba19c2 100644 --- a/spark/core/src/test/java/zingg/spark/core/executor/ZinggSparkTester.java +++ b/spark/core/src/test/java/zingg/spark/core/executor/ZinggSparkTester.java @@ -46,6 +46,7 @@ public static void setup() { .getOrCreate(); ctx = new JavaSparkContext(spark.sparkContext()); JavaSparkContext.jarOfClass(IZingg.class); + ctx.setCheckpointDir("/tmp/checkpoint"); args = new Arguments(); zsCTX = new ZinggSparkContext(); zsCTX.init(spark);