correct errors in Spark

EUBra-BIGSEA · Jul 28, 2017 · f85cafe · f85cafe
1 parent 634e155
commit f85cafe
Show file tree

Hide file tree

Showing 15 changed files with 76 additions and 93 deletions.
diff --git a/Spark/derby.log b/Spark/derby.log
diff --git a/Spark/metastore_db/db.lck b/Spark/metastore_db/db.lck
diff --git a/Spark/metastore_db/log/log.ctrl b/Spark/metastore_db/log/log.ctrl
diff --git a/Spark/metastore_db/log/log1.dat b/Spark/metastore_db/log/log1.dat
diff --git a/Spark/metastore_db/log/logmirror.ctrl b/Spark/metastore_db/log/logmirror.ctrl
diff --git a/Spark/metastore_db/seg0/c180.dat b/Spark/metastore_db/seg0/c180.dat
diff --git a/Spark/metastore_db/seg0/c191.dat b/Spark/metastore_db/seg0/c191.dat
diff --git a/Spark/metastore_db/seg0/c1a1.dat b/Spark/metastore_db/seg0/c1a1.dat
diff --git a/Spark/metastore_db/seg0/c1b1.dat b/Spark/metastore_db/seg0/c1b1.dat
diff --git a/Spark/metastore_db/seg0/c230.dat b/Spark/metastore_db/seg0/c230.dat
diff --git a/Spark/metastore_db/seg0/c241.dat b/Spark/metastore_db/seg0/c241.dat
diff --git a/Spark/wc_spark.py b/Spark/wc_spark.py
@@ -1,7 +1,7 @@
 from pyspark import SparkConf, SparkContext
 import sys
 
-conf = SparkConf().setMaster("local").setAppName("My App")
+conf = SparkConf().setAppName("My App")
 sc = SparkContext(conf = conf)
 
 if len(sys.argv)<2 :

diff --git a/WordCount/WordCount.jar b/WordCount/WordCount.jar
diff --git a/spark_test.sh b/spark_test.sh
@@ -4,19 +4,19 @@ source vars.sh
 
 echo "##### Compute an approximation of Pi in python"
 cd Spark
-spark-submit --executor-memory 256M --num-executors 1 --master mesos://$MESOSCLUSTER:$MESOSPORT spark-pi.py
+spark-submit --executor-memory 512M --total-executor-cores 2 --master mesos://$MESOSCLUSTER:$MESOSPORT spark-pi.py
 
 echo "############ Spark job via spark-submit, with interaction with HDFS"
 echo "##### Compute wordcount with Spark"
 hdfs dfs -copyFromLocal palabras.txt $HDFSHOMEDIR/WordCount
-spark-submit --executor-memory 256M --num-executors 1 --master mesos://$MESOSCLUSTER:$MESOSPORT wc_spark.py hdfs://$HDFSINTERNAL:$HDFSPORT$HDFSHOMEDIR/WordCount/palabras.txt
+spark-submit --executor-memory 512M --total-executor-cores 4 --master mesos://$MESOSCLUSTER:$MESOSPORT wc_spark.py hdfs://$HDFSINTERNAL:$HDFSPORT$HDFSHOMEDIR/WordCount/palabras.txt
 
 sleep $PAUSETIME
 
 
 echo "############ Spark execution via spark-shell, without interaction with HDFS"
 echo "##### Compute an approximation of Pi in python"
 
-spark-shell --executor-memory=256M --master mesos://$MESOSCLUSTER:$MESOSPORT <spark_pi.scala
+spark-shell --executor-memory=512M --master mesos://$MESOSCLUSTER:$MESOSPORT <spark_pi.scala
 cd ..
 
diff --git a/vars.sh → vars_template.sh b/vars.sh → vars_template.sh
@@ -1,5 +1,5 @@
-MESOSCLUSTER=158.42.104.238
-HDFSINTERNAL=10.0.0.22
+MESOSCLUSTER=
+HDFSINTERNAL=
 HDFSPORT=9000
 MESOSPORT=5050
 CHRONOSPORT=4400