Skip to content

Commit

Permalink
Scaled jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
ubuntu committed Sep 8, 2016
1 parent dc0f53b commit 02fad46
Show file tree
Hide file tree
Showing 33 changed files with 2,028 additions and 173 deletions.
166 changes: 83 additions & 83 deletions Spark/derby.log
View file

Large diffs are not rendered by default.

Binary file modified Spark/metastore_db/db.lck
View file
Binary file not shown.
Binary file modified Spark/metastore_db/log/log.ctrl
View file
Binary file not shown.
Binary file modified Spark/metastore_db/log/log1.dat
View file
Binary file not shown.
Binary file modified Spark/metastore_db/log/logmirror.ctrl
View file
Binary file not shown.
Binary file modified Spark/metastore_db/seg0/c180.dat
View file
Binary file not shown.
Binary file modified Spark/metastore_db/seg0/c191.dat
View file
Binary file not shown.
Binary file modified Spark/metastore_db/seg0/c1a1.dat
View file
Binary file not shown.
Binary file modified Spark/metastore_db/seg0/c1b1.dat
View file
Binary file not shown.
21 changes: 21 additions & 0 deletions Spark/spark-pi-long.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/usr/bin/python

import sys, random
from pyspark import SparkConf, SparkContext

if len(sys.argv)<2 :
NUM_SAMPLES=100000000
else:
NUM_SAMPLES=int(sys.argv[1])

conf = SparkConf().setMaster("local").setAppName("My App")
sc = SparkContext(conf = conf)
def sample(p):
x, y = random.random(), random.random()
return 1 if x*x + y*y < 1 else 0

count = sc.parallelize(xrange(0, NUM_SAMPLES)).map(sample) \
.reduce(lambda a, b: a + b)
print "Pi is roughly %f" % (4.0 * count / NUM_SAMPLES)


Binary file modified WordCount/WordCount.jar
View file
Binary file not shown.
6 changes: 6 additions & 0 deletions WordCount/doit.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
cd WordCount
hdfs dfs -mkdir $HDFSHOMEDIR/WordCount
hdfs dfs -copyFromLocal quijote.txt $HDFSHOMEDIR/WordCount
./compile.sh WordCount.java
hadoop jar WordCount.jar org.apache.hadoop.examples.WordCount $HDFSHOMEDIR/WordCount/quijote.txt $HDFSHOMEDIR/WordCount/output
cd ..
39 changes: 39 additions & 0 deletions chronos_test.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#!/bin/bash

set -x
source vars.sh

echo "##### Simple job, job embedded in the command argument"
echo "# Submission of the framework"
curl -L -H 'Content-Type: application/json' -X POST -d '{ "schedule": "R5/2016-06-01T07:15:00Z/PT2S", "name": "BIGSEA_simpletest", "epsilon": "PT15M", "command": "echo 'Run' >> /tmp/simple_test.out", "owner": "[email protected]", "async": false }' $MESOSCLUSTER:$CHRONOSPORT/scheduler/iso8601
echo "# Run job"
curl -L -X PUT $MESOSCLUSTER:$CHRONOSPORT/scheduler/job/BIGSEA_simpletest

sleep $PAUSETIME


echo "##### Simple container job, job embedded in the command argument and no volumes"
echo "# Submission of the framework"
curl -L -H 'Content-Type: application/json' -X POST -d '{ "schedule": "R5/2016-06-01T07:45:00Z/PT2S", "name": "BIGSEA_dockertest", "container": { "type": "DOCKER", "image": "ubuntu", "network": "BRIDGE" }, "cpus": "0.5", "mem": "512", "uris": [], "command": "while sleep 10; do date =u %T; done" }' $MESOSCLUSTER:$CHRONOSPORT/scheduler/iso8601
echo "# Run job"
curl -L -X PUT $MESOSCLUSTER:$CHRONOSPORT/scheduler/job/BIGSEA_dockertest

sleep $PAUSETIME

echo "##### Container-specific job, containers mounts a volume from the shared directory where data and executable files are found"
echo "# Submission of the framework"
curl -i -L -H 'Content-Type: application/json' -X POST -d@"container_job_volume.json" $MESOSCLUSTER:$CHRONOSPORT/scheduler/iso8601
echo "# Run job"
curl -L -X PUT $MESOSCLUSTER:$CHRONOSPORT/scheduler/job/docker_volume

sleep $PAUSETIME

echo "##### Long-time executing container job that runs the linpack benchmark several times. Used to test vertical elasticity"
echo "# Submission of the framework"
curl -i -L -H 'Content-Type: application/json' -X POST -d@"linpack.json" $MESOSCLUSTER:$CHRONOSPORT/scheduler/iso8601
echo "# Run job"
curl -L -X PUT $MESOSCLUSTER:$CHRONOSPORT/scheduler/job/linpack_test
echo "# It will long for a while. You can change the allocation of resources with the following command:"
echo "curl -i -L -H 'Content-Type: application/json' -X POST -d@\"linpack_scaled.json\" $MESOSCLUSTER:$CHRONOSPORT/scheduler/iso8601"


2 changes: 1 addition & 1 deletion data/data_A
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Created by Octave 3.8.1, Thu Aug 04 11:17:36 2016 UTC <root@vnode4.localdomain>
# Created by Octave 3.8.1, Wed Sep 07 17:26:13 2016 UTC <root@vnode6.localdomain>
# name: A
# type: matrix
# rows: 7
Expand Down
1,258 changes: 1,258 additions & 0 deletions data/marathon_out.txt
View file

Large diffs are not rendered by default.

34 changes: 34 additions & 0 deletions data/newoutput.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
Rolled Double Precision Linpack (1000)

Wed Sep 7 14:44:29 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.17 0.00 0.17 480980 0.00 3.11
0.13 0.00 0.13 639783 0.00 2.34
0.19 0.00 0.19 435260 0.00 3.44
0.15 0.00 0.15 564058 0.00 2.65
times for array with leading dimension of1000
0.14 0.00 0.14 604087 0.00 2.48
0.14 0.00 0.14 596415 0.00 2.51
0.14 0.00 0.14 597849 0.00 2.50
0.15 0.00 0.15 567409 0.00 2.64
Rolled Double Precision 564058 Kflops ; 1000 Reps
Rolled Double Precision Linpack (1000)

Wed Sep 7 14:52:32 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.18 0.00 0.19 451129 0.00 3.32
0.14 0.00 0.14 599186 0.00 2.50
0.14 0.00 0.14 606310 0.00 2.47
0.14 0.00 0.14 617151 0.00 2.43
times for array with leading dimension of1000
0.13 0.00 0.13 624820 0.00 2.40
0.13 0.00 0.13 623997 0.00 2.40
0.13 0.00 0.13 624890 0.00 2.40
0.13 0.00 0.14 620944 0.00 2.41
Rolled Double Precision 617151 Kflops ; 1000 Reps
51 changes: 51 additions & 0 deletions data/newoutput_old.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
Rolled Double Precision Linpack (1000)

Wed Sep 7 11:19:53 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.18 0.00 0.19 450887 0.00 3.32
0.15 0.00 0.15 540993 0.00 2.77
0.14 0.00 0.14 598788 0.00 2.50
0.14 0.00 0.14 594136 0.00 2.52
times for array with leading dimension of1000
0.13 0.00 0.13 639276 0.00 2.34
0.12 0.00 0.12 679142 0.00 2.20
0.12 0.00 0.12 681156 0.00 2.20
0.14 0.00 0.14 604124 0.00 2.48
Rolled Double Precision 594136 Kflops ; 1000 Reps
Rolled Double Precision Linpack (1000)

Wed Sep 7 11:26:14 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.18 0.00 0.19 451013 0.00 3.32
0.17 0.00 0.17 490308 0.00 3.05
0.15 0.00 0.15 542530 0.00 2.76
0.14 0.00 0.14 581575 0.00 2.57
times for array with leading dimension of1000
0.18 0.00 0.18 464858 0.00 3.22
0.12 0.00 0.12 678219 0.00 2.21
0.12 0.00 0.12 676763 0.00 2.21
0.14 0.00 0.15 576805 0.00 2.60
Rolled Double Precision 576805 Kflops ; 1000 Reps
Rolled Double Precision Linpack (1000)

Wed Sep 7 11:34:14 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.19 0.00 0.19 448578 0.00 3.34
0.18 0.00 0.18 470416 0.00 3.18
0.14 0.00 0.14 586834 0.00 2.55
0.15 0.00 0.15 573933 0.00 2.61
times for array with leading dimension of1000
0.17 0.00 0.17 494717 0.00 3.03
0.12 0.00 0.12 672269 0.00 2.23
0.12 0.00 0.12 677907 0.00 2.21
0.15 0.00 0.15 574429 0.00 2.61
Rolled Double Precision 573933 Kflops ; 1000 Reps
102 changes: 102 additions & 0 deletions data/newoutput_success.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
Rolled Double Precision Linpack (1000)

Wed Sep 7 11:19:53 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.18 0.00 0.19 450887 0.00 3.32
0.15 0.00 0.15 540993 0.00 2.77
0.14 0.00 0.14 598788 0.00 2.50
0.14 0.00 0.14 594136 0.00 2.52
times for array with leading dimension of1000
0.13 0.00 0.13 639276 0.00 2.34
0.12 0.00 0.12 679142 0.00 2.20
0.12 0.00 0.12 681156 0.00 2.20
0.14 0.00 0.14 604124 0.00 2.48
Rolled Double Precision 594136 Kflops ; 1000 Reps
Rolled Double Precision Linpack (1000)

Wed Sep 7 11:26:14 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.18 0.00 0.19 451013 0.00 3.32
0.17 0.00 0.17 490308 0.00 3.05
0.15 0.00 0.15 542530 0.00 2.76
0.14 0.00 0.14 581575 0.00 2.57
times for array with leading dimension of1000
0.18 0.00 0.18 464858 0.00 3.22
0.12 0.00 0.12 678219 0.00 2.21
0.12 0.00 0.12 676763 0.00 2.21
0.14 0.00 0.15 576805 0.00 2.60
Rolled Double Precision 576805 Kflops ; 1000 Reps
Rolled Double Precision Linpack (1000)

Wed Sep 7 11:34:14 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.19 0.00 0.19 448578 0.00 3.34
0.18 0.00 0.18 470416 0.00 3.18
0.14 0.00 0.14 586834 0.00 2.55
0.15 0.00 0.15 573933 0.00 2.61
times for array with leading dimension of1000
0.17 0.00 0.17 494717 0.00 3.03
0.12 0.00 0.12 672269 0.00 2.23
0.12 0.00 0.12 677907 0.00 2.21
0.15 0.00 0.15 574429 0.00 2.61
Rolled Double Precision 573933 Kflops ; 1000 Reps
Rolled Double Precision Linpack (1000)

Wed Sep 7 11:42:12 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.19 0.00 0.19 430763 0.00 3.48
0.16 0.00 0.16 510538 0.00 2.93
0.16 0.00 0.16 532840 0.00 2.81
0.15 0.00 0.15 572186 0.00 2.62
times for array with leading dimension of1000
0.14 0.00 0.14 582261 0.00 2.57
0.14 0.00 0.14 590376 0.00 2.54
0.14 0.00 0.14 593204 0.00 2.52
0.15 0.00 0.15 572088 0.00 2.62
Rolled Double Precision 572088 Kflops ; 1000 Reps
Rolled Double Precision Linpack (1000)

Wed Sep 7 11:52:44 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.17 0.00 0.17 490532 0.00 3.05
0.14 0.00 0.14 607338 0.00 2.46
0.15 0.00 0.15 558078 0.00 2.68
0.13 0.00 0.13 626009 0.00 2.39
times for array with leading dimension of1000
0.13 0.00 0.13 636475 0.00 2.35
0.13 0.00 0.14 618007 0.00 2.42
0.13 0.00 0.13 636171 0.00 2.35
0.13 0.00 0.13 624380 0.00 2.40
Rolled Double Precision 624380 Kflops ; 1000 Reps
Rolled Double Precision Linpack (1000)

Wed Sep 7 11:58:16 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.19 0.00 0.19 439638 0.00 3.41
0.15 0.00 0.15 542779 0.00 2.76
0.19 0.00 0.19 448438 0.00 3.34
0.15 0.00 0.15 574736 0.00 2.60
times for array with leading dimension of1000
0.18 0.00 0.18 471278 0.00 3.18
0.14 0.00 0.14 601754 0.00 2.49
0.14 0.00 0.14 596666 0.00 2.51
0.14 0.00 0.15 576867 0.00 2.60
Rolled Double Precision 574736 Kflops ; 1000 Reps
83 changes: 83 additions & 0 deletions data/success_execution_chronos.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
Rolled Double Precision Linpack (10)

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.19 0.00 0.19 447890 0.00 3.34
0.17 0.00 0.17 481920 0.00 3.11
0.18 0.00 0.18 454482 0.00 3.29
0.12 0.00 0.12 673282 0.00 2.22
times for array with leading dimension of1000
0.12 0.00 0.12 679886 0.00 2.20
0.12 0.00 0.12 678587 0.00 2.21
0.12 0.00 0.12 680294 0.00 2.20
0.12 0.00 0.12 673631 0.00 2.22
Rolled Double Precision 673282 Kflops ; 1000 Reps
Rolled Double Precision Linpack (10)

Wed Sep 7 10:43:14 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.18 0.00 0.19 451859 0.00 3.31
0.13 0.00 0.13 622667 0.00 2.40
0.13 0.00 0.13 632404 0.00 2.37
0.14 0.00 0.14 610196 0.00 2.45
times for array with leading dimension of1000
0.13 0.00 0.13 638677 0.00 2.34
0.13 0.00 0.13 635650 0.00 2.36
0.13 0.00 0.13 629900 0.00 2.38
0.13 0.00 0.13 627081 0.00 2.39
Rolled Double Precision 610196 Kflops ; 10 Reps
Rolled Double Precision Linpack (10)

Wed Sep 7 10:45:17 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.17 0.00 0.17 493340 0.00 3.03
0.12 0.00 0.12 680686 0.00 2.20
0.12 0.00 0.12 680675 0.00 2.20
0.13 0.00 0.13 621037 0.00 2.41
times for array with leading dimension of1000
0.13 0.00 0.13 628733 0.00 2.38
0.13 0.00 0.13 630577 0.00 2.37
0.13 0.00 0.13 633608 0.00 2.36
0.13 0.00 0.13 628187 0.00 2.38
Rolled Double Precision 621037 Kflops ; 10 Reps
Rolled Double Precision Linpack (10)

Wed Sep 7 10:47:14 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.18 0.00 0.19 451730 0.00 3.31
0.18 0.00 0.18 455908 0.00 3.28
0.17 0.00 0.17 493073 0.00 3.04
0.14 0.00 0.14 601680 0.00 2.49
times for array with leading dimension of1000
0.14 0.00 0.14 606534 0.00 2.47
0.13 0.00 0.13 634534 0.00 2.36
0.13 0.00 0.13 631684 0.00 2.37
0.13 0.00 0.13 628568 0.00 2.38
Rolled Double Precision 601680 Kflops ; 10 Reps
Rolled Double Precision Linpack (10)

Wed Sep 7 10:49:20 2016

times are reported for matrices of order 500
dgefa dgesl total kflops unit ratio
times for array with leading dimension of 1001
0.19 0.00 0.19 450303 0.00 3.32
0.16 0.00 0.16 526363 0.00 2.84
0.12 0.00 0.12 679610 0.00 2.20
0.13 0.00 0.13 665642 0.00 2.25
times for array with leading dimension of1000
0.13 0.00 0.13 648574 0.00 2.31
0.13 0.00 0.13 655839 0.00 2.28
0.12 0.00 0.12 677688 0.00 2.21
0.13 0.00 0.13 660419 0.00 2.27
Rolled Double Precision 660419 Kflops ; 10 Reps
12 changes: 12 additions & 0 deletions hdfs_test.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

source vars.sh
echo "##### Copy WordCount data and program, compile and run it"

cd WordCount
hdfs dfs -mkdir $HDFSHOMEDIR/WordCount
hdfs dfs -copyFromLocal quijote.txt $HDFSHOMEDIR/WordCount
./compile.sh WordCount.java
hadoop jar WordCount.jar org.apache.hadoop.examples.WordCount $HDFSHOMEDIR/WordCount/quijote.txt $HDFSHOMEDIR/WordCount/output
cd ..

2 changes: 1 addition & 1 deletion launch_marathon.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ if [ "$#" -ne 1 ]; then
exit 1;
fi

curl -i -L -H 'Content-Type: application/json' -X POST -d@"$@" 158.42.105.14:8080/v2/apps
curl -i -L -H 'Content-Type: application/json' -X POST -d@"$@" 158.42.104.229:8080/v2/apps?force=true


21 changes: 21 additions & 0 deletions linpack2_chronos.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"schedule": "R5//PT4M",
"name": "linpack_test",
"container": {
"type": "DOCKER",
"image": "eubrabigsea/r-client",
"forcePullImage": true,
"volumes": [
{
"containerPath": "/data/",
"hostPath": "/home/ubuntu/testing/data",
"mode": "RW"
} ]
},
"cpus": "0.5",
"mem": "256",
"command": "/linpack /data/newoutput.txt 1000",
"owner": "[email protected]",
"async": "false"
}

Loading

0 comments on commit 02fad46

Please sign in to comment.