From 41ca9703fd05ee8d7c3736522fed2d4e5a660150 Mon Sep 17 00:00:00 2001
From: Erick Lavoie <erick.lavoie@epfl.ch>
Date: Tue, 15 Mar 2022 14:28:15 +0100
Subject: [PATCH] Added option to disable dynamic allocation on the cluster for
 performance tests

---
 README.md      | 2 ++
 timeCluster.sh | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index a9f1daa..6191a8b 100644
--- a/README.md
+++ b/README.md
@@ -123,6 +123,8 @@ On ````ml-25m````:
 
 You can vary the number of executors used locally by using ````local[X]```` with X being an integer representing the number of cores you want to use locally.
 
+You can vary the number of executors on the cluster by disabling dynamic allocation with ````--conf "spark.dynamicAllocation.enabled=false"````, using ````--master yarn````, and setting the number of executors with ````--num-executors X````.
+
 ### Personalized
 
 ````
diff --git a/timeCluster.sh b/timeCluster.sh
index 597d3c3..c73d0ad 100755
--- a/timeCluster.sh
+++ b/timeCluster.sh
@@ -11,6 +11,6 @@ source ./config.sh
 echo "------------------- DISTRIBUTED ---------------------" >> $LOGS
 sbt assembly
 # 1 Executor
-spark-submit --class distributed.DistributedBaseline --master $SPARKMASTER --num-executors 1 target/scala-2.11/m1_yourid-assembly-1.0.jar  --train $ML25Mr2train --test $ML25Mr2test --separator , --json $RUN/distributed-25m-1.json --num_measurements 3 2>&1 >>$LOGS
+spark-submit --class distributed.DistributedBaseline --master $SPARKMASTER --num-executors 1 --conf "spark.dynamicAllocation.enabled=false" target/scala-2.11/m1_yourid-assembly-1.0.jar  --train $ML25Mr2train --test $ML25Mr2test --separator , --json $RUN/distributed-25m-1.json --num_measurements 3 2>&1 >>$LOGS
 # 4 Executors
-spark-submit --class distributed.DistributedBaseline --master $SPARKMASTER --num-executors 4 target/scala-2.11/m1_yourid-assembly-1.0.jar  --train $ML25Mr2train --test $ML25Mr2test --separator , --json $RUN/distributed-25m-4.json --num_measurements 3 2>&1 >>$LOGS
+spark-submit --class distributed.DistributedBaseline --master $SPARKMASTER --num-executors 4 --conf "spark.dynamicAllocation.enabled=false" target/scala-2.11/m1_yourid-assembly-1.0.jar  --train $ML25Mr2train --test $ML25Mr2test --separator , --json $RUN/distributed-25m-4.json --num_measurements 3 2>&1 >>$LOGS
-- 
GitLab