diff --git a/README.md b/README.md index a9f1daade375c4403d0d4bfa3c0af60ebd09a8e5..6191a8ba4e5b1af56830aa92bcf6473a466c39a7 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,8 @@ On ````ml-25m````: You can vary the number of executors used locally by using ````local[X]```` with X being an integer representing the number of cores you want to use locally. +You can vary the number of executors on the cluster by disabling dynamic allocation with ````--conf "spark.dynamicAllocation.enabled=false"````, using ````--master yarn````, and setting the number of executors with ````--num-executors X````. + ### Personalized ```` diff --git a/timeCluster.sh b/timeCluster.sh index 597d3c3ca61779664335453be70130d21d262d60..c73d0ad20226fc4dca6083b5dc0234f9b2460a4c 100755 --- a/timeCluster.sh +++ b/timeCluster.sh @@ -11,6 +11,6 @@ source ./config.sh echo "------------------- DISTRIBUTED ---------------------" >> $LOGS sbt assembly # 1 Executor -spark-submit --class distributed.DistributedBaseline --master $SPARKMASTER --num-executors 1 target/scala-2.11/m1_yourid-assembly-1.0.jar --train $ML25Mr2train --test $ML25Mr2test --separator , --json $RUN/distributed-25m-1.json --num_measurements 3 2>&1 >>$LOGS +spark-submit --class distributed.DistributedBaseline --master $SPARKMASTER --num-executors 1 --conf "spark.dynamicAllocation.enabled=false" target/scala-2.11/m1_yourid-assembly-1.0.jar --train $ML25Mr2train --test $ML25Mr2test --separator , --json $RUN/distributed-25m-1.json --num_measurements 3 2>&1 >>$LOGS # 4 Executors -spark-submit --class distributed.DistributedBaseline --master $SPARKMASTER --num-executors 4 target/scala-2.11/m1_yourid-assembly-1.0.jar --train $ML25Mr2train --test $ML25Mr2test --separator , --json $RUN/distributed-25m-4.json --num_measurements 3 2>&1 >>$LOGS +spark-submit --class distributed.DistributedBaseline --master $SPARKMASTER --num-executors 4 --conf "spark.dynamicAllocation.enabled=false" target/scala-2.11/m1_yourid-assembly-1.0.jar --train $ML25Mr2train --test $ML25Mr2test --separator , --json $RUN/distributed-25m-4.json --num_measurements 3 2>&1 >>$LOGS