diff --git a/figures/d-clique-mnist-clique-avg.png b/figures/d-clique-mnist-clique-avg.png
index e15a177a7bebd6f9716f8a47a816541912e16af5..e5aab71d7e8e4bda14544670466a567b518343ac 100644
Binary files a/figures/d-clique-mnist-clique-avg.png and b/figures/d-clique-mnist-clique-avg.png differ
diff --git a/figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-linear.png b/figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-linear.png
new file mode 100644
index 0000000000000000000000000000000000000000..ea303d8c7af90e2488b317a01cc640d880328511
Binary files /dev/null and b/figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-linear.png differ
diff --git a/figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-super-linear.png b/figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-super-linear.png
new file mode 100644
index 0000000000000000000000000000000000000000..ea35c410e3d986dc0df2abf2f4edf1f0b9f624a3
Binary files /dev/null and b/figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-super-linear.png differ
diff --git a/figures/d-cliques-cifar10-clique-clustering-fcc-minus-1-edge.png b/figures/d-cliques-cifar10-clique-clustering-fcc-minus-1-edge.png
new file mode 100644
index 0000000000000000000000000000000000000000..37e596b8870f54734e49901bac924739d29c5159
Binary files /dev/null and b/figures/d-cliques-cifar10-clique-clustering-fcc-minus-1-edge.png differ
diff --git a/figures/d-cliques-cifar10-clique-clustering-fcc-minus-5-edges.png b/figures/d-cliques-cifar10-clique-clustering-fcc-minus-5-edges.png
new file mode 100644
index 0000000000000000000000000000000000000000..9bd24b6bcfa066badafe25d8360c9c682f88a5ec
Binary files /dev/null and b/figures/d-cliques-cifar10-clique-clustering-fcc-minus-5-edges.png differ
diff --git a/figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png b/figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png
index 4ecd8427e1c3dc504b81deec70041abe0491de91..fd1281c31729eb953c3396f383d4d9024d28bb65 100644
Binary files a/figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png and b/figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png differ
diff --git a/figures/d-cliques-cifar10-momentum-non-iid-clique-avg-effect.png b/figures/d-cliques-cifar10-momentum-non-iid-clique-avg-effect.png
index 93cc87ed2235c95478cfb26d3d1dfaec30dd2f6c..1966e71b76e2641d7cba376a3a13393e4021ea9f 100644
Binary files a/figures/d-cliques-cifar10-momentum-non-iid-clique-avg-effect.png and b/figures/d-cliques-cifar10-momentum-non-iid-clique-avg-effect.png differ
diff --git a/figures/d-cliques-cifar10-momentum-non-iid-effect.png b/figures/d-cliques-cifar10-momentum-non-iid-effect.png
index 71ba70479ace7ecc375e3a6f1bd661476b36c908..e130e30e864b98ebbbfd6774fc2cdee3eb240e72 100644
Binary files a/figures/d-cliques-cifar10-momentum-non-iid-effect.png and b/figures/d-cliques-cifar10-momentum-non-iid-effect.png differ
diff --git a/figures/d-cliques-mnist-1000-nodes-comparison-linear.png b/figures/d-cliques-mnist-1000-nodes-comparison-linear.png
new file mode 100644
index 0000000000000000000000000000000000000000..7ae6094fa096f52b7f0b057ac7d8026239872868
Binary files /dev/null and b/figures/d-cliques-mnist-1000-nodes-comparison-linear.png differ
diff --git a/figures/d-cliques-mnist-1000-nodes-comparison-super-linear.png b/figures/d-cliques-mnist-1000-nodes-comparison-super-linear.png
new file mode 100644
index 0000000000000000000000000000000000000000..346c5ff32d41929bd0235c64addb486816f45c1d
Binary files /dev/null and b/figures/d-cliques-mnist-1000-nodes-comparison-super-linear.png differ
diff --git a/figures/d-cliques-mnist-clique-clustering-fcc-minus-1-edge.png b/figures/d-cliques-mnist-clique-clustering-fcc-minus-1-edge.png
new file mode 100644
index 0000000000000000000000000000000000000000..baa4b93adc9e89a5e6a002eb1f6b7232c69f322e
Binary files /dev/null and b/figures/d-cliques-mnist-clique-clustering-fcc-minus-1-edge.png differ
diff --git a/figures/d-cliques-mnist-clique-clustering-fcc-minus-5-edges.png b/figures/d-cliques-mnist-clique-clustering-fcc-minus-5-edges.png
new file mode 100644
index 0000000000000000000000000000000000000000..213c45cb6e2ef66ed16c5d7ce8ff4ce280540cdb
Binary files /dev/null and b/figures/d-cliques-mnist-clique-clustering-fcc-minus-5-edges.png differ
diff --git a/figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png b/figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png
index ff066d43b0f69ca31037362f16c2f2dd3b65018f..1e25a25bb1869801caaa9e03301377a4a3799809 100644
Binary files a/figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png and b/figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png differ
diff --git a/figures/d-cliques-mnist-vs-fully-connected.png b/figures/d-cliques-mnist-vs-fully-connected.png
index 8721f3fd8332d2acb0f65751f18cdf4417e52206..1b82a74aae176ac915dcc5a79c1e574d7ec722b7 100644
Binary files a/figures/d-cliques-mnist-vs-fully-connected.png and b/figures/d-cliques-mnist-vs-fully-connected.png differ
diff --git a/figures/fully-connected-IID-vs-non-IID.png b/figures/fully-connected-IID-vs-non-IID.png
index 77e8914712046e8784c1fa23cf018c545a830713..d6344df209cfb0af9fc94bc3d955cea563003062 100644
Binary files a/figures/fully-connected-IID-vs-non-IID.png and b/figures/fully-connected-IID-vs-non-IID.png differ
diff --git a/figures/grid-IID-vs-non-IID.png b/figures/grid-IID-vs-non-IID.png
index b809feab14c57cc1da97954c8e9088ffdfadb155..9cb5c3df5c12c132846e6bbf67d3c24a24855d93 100644
Binary files a/figures/grid-IID-vs-non-IID.png and b/figures/grid-IID-vs-non-IID.png differ
diff --git a/figures/ring-IID-vs-non-IID.png b/figures/ring-IID-vs-non-IID.png
index 6faa52fd396b8005ad428867d135175d891779ac..997270059b59122473ec5cbd9736cd1eae3d6b30 100644
Binary files a/figures/ring-IID-vs-non-IID.png and b/figures/ring-IID-vs-non-IID.png differ
diff --git a/main.tex b/main.tex
index a3b662489b3643fff909627ee5ec641ecca932f6..d694c41118655f6c5cfffee9c62dee2798e419b4 100644
--- a/main.tex
+++ b/main.tex
@@ -136,7 +136,7 @@ model with their neighbors. In this paper, we address the following question:
\centering
% From directory results/mnist
- % python ../../../learn-topology/tools/plot_convergence.py ring/iid/all/2021-03-30-16:07:06-CEST ring/non-iid/all/2021-03-30-16:07:03-CEST --add-min-max --legend 'lower right' --yaxis test-accuracy --labels '100 nodes IID' '100 nodes non-IID' --save-figure ../../figures/ring-IID-vs-non-IID.png --font-size 20
+ % python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py ring/iid/all/2021-03-30-16:07:06-CEST ring/non-iid/all/2021-03-30-16:07:03-CEST --add-min-max --legend 'lower right' --yaxis test-accuracy --labels '100 nodes IID' '100 nodes non-IID' --save-figure ../../figures/ring-IID-vs-non-IID.png --font-size 20 --linestyles 'solid' 'dashed'
\begin{subfigure}[b]{0.31\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/ring-IID-vs-non-IID}
@@ -144,7 +144,7 @@ model with their neighbors. In this paper, we address the following question:
\end{subfigure}
\quad
% From directory results/mnist
- % python ../../../learn-topology/tools/plot_convergence.py grid/iid/all/2021-03-30-16:07:01-CEST grid/non-iid/all/2021-03-30-16:06:59-CEST --add-min-max --legend 'lower right' --yaxis test-accuracy --labels '100 nodes IID' '100 nodes non-IID' --save-figure ../../figures/grid-IID-vs-non-IID.png --font-size 20
+ % python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py grid/iid/all/2021-03-30-16:07:01-CEST grid/non-iid/all/2021-03-30-16:06:59-CEST --add-min-max --legend 'lower right' --yaxis test-accuracy --labels '100 nodes IID' '100 nodes non-IID' --save-figure ../../figures/grid-IID-vs-non-IID.png --font-size 20 --linestyles 'solid' 'dashed'
\begin{subfigure}[b]{0.31\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/grid-IID-vs-non-IID}
@@ -152,7 +152,7 @@ model with their neighbors. In this paper, we address the following question:
\end{subfigure}
\quad
% From directory results/mnist
- % python ../../../learn-topology/tools/plot_convergence.py fully-connected/iid/all/2021-03-30-16:07:20-CEST fully-connected/all/2021-03-10-09:25:19-CET --add-min-max --legend 'lower right' --yaxis test-accuracy --labels '100 nodes IID' '100 nodes non-IID' --save-figure ../../figures/fully-connected-IID-vs-non-IID.png --font-size 20
+ % python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py fully-connected/iid/all/2021-03-30-16:07:20-CEST fully-connected/all/2021-03-10-09:25:19-CET --add-min-max --legend 'lower right' --yaxis test-accuracy --labels '100 nodes IID' '100 nodes non-IID' --save-figure ../../figures/fully-connected-IID-vs-non-IID.png --font-size 20 --linestyles 'solid' 'dashed'
\begin{subfigure}[b]{0.31\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/fully-connected-IID-vs-non-IID}
@@ -450,7 +450,7 @@ topology very well-suited to decentralized federated learning.
\end{subfigure}
\hfill
% To regenerate figure, from results/mnist
- % python ../../../learn-topology/tools/plot_convergence.py fully-connected/all/2021-03-10-09:25:19-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-12-11:12:49-CET --add-min-max --yaxis test-accuracy --ymin 80 --ymax 92.5 --labels '100 nodes non-IID fully-connected' '100 nodes non-IID d-cliques' --save-figure ../../figures/d-cliques-mnist-vs-fully-connected.png --legend 'lower right' --font-size 16
+ % python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py fully-connected/all/2021-03-10-09:25:19-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-12-11:12:49-CET --add-min-max --yaxis test-accuracy --ymin 80 --ymax 92.5 --labels '100 nodes non-IID fully-connected' '100 nodes non-IID d-cliques' --save-figure ../../figures/d-cliques-mnist-vs-fully-connected.png --legend 'lower right' --font-size 16 --linestyles 'solid' 'dashed'
\begin{subfigure}[b]{0.54\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/d-cliques-mnist-vs-fully-connected.png}
@@ -539,7 +539,7 @@ averaging step as in the original version.
\end{algorithm}
% To regenerate figure, from results/mnist:
-% python ../../../learn-topology/tools/plot_convergence.py fully-connected/all/2021-03-10-09:25:19-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-12-11:12:49-CET no-init/fully-connected-cliques/all/2021-03-12-11:12:01-CET --add-min-max --yaxis test-accuracy --labels '100 nodes non-IID fully-connected' '100 nodes non-IID d-cliques w/o clique avg.' '100 nodes non-IID w/ clique avg.' --legend 'lower right' --ymin 89 --ymax 92.5 --font-size 13 --save-figure ../../figures/d-clique-mnist-clique-avg.png
+% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py fully-connected/all/2021-03-10-09:25:19-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-12-11:12:49-CET no-init/fully-connected-cliques/all/2021-03-12-11:12:01-CET --add-min-max --yaxis test-accuracy --labels '100 nodes non-IID fully-connected' '100 nodes non-IID d-cliques w/o clique avg.' '100 nodes d-cliques non-IID w/ clique avg.' --legend 'lower right' --ymin 89 --ymax 92.5 --font-size 13 --save-figure ../../figures/d-clique-mnist-clique-avg.png --linestyles 'solid' 'dashed' 'dotted'
\begin{figure}[t]
\centering
\includegraphics[width=0.55\textwidth]{figures/d-clique-mnist-clique-avg}
@@ -571,7 +571,7 @@ IID node with momentum.
\begin{figure}[t]
\centering
% To regenerate figure, from results/cifar10
- % python ../../../learn-topology/tools/plot_convergence.py 1-node-iid/all/2021-03-10-13:52:58-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-13-18:34:35-CET no-init-no-clique-avg-no-momentum/fully-connected-cliques/all/2021-03-26-13:47:35-CET/ --legend 'upper right' --add-min-max --labels '1-node IID w/ momentum' '100 nodes non-IID d-cliques w/ momentum' '100 nodes non-IID d-cliques w/o momentum' --font-size 14 --yaxis test-accuracy --save-figure ../../figures/d-cliques-cifar10-momentum-non-iid-effect.png --ymax 100
+ % python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py 1-node-iid/all/2021-03-10-13:52:58-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-13-18:34:35-CET no-init-no-clique-avg-no-momentum/fully-connected-cliques/all/2021-03-26-13:47:35-CET/ --legend 'upper right' --add-min-max --labels '1-node IID w/ momentum' '100 nodes non-IID d-cliques w/ momentum' '100 nodes non-IID d-cliques w/o momentum' --font-size 14 --yaxis test-accuracy --save-figure ../../figures/d-cliques-cifar10-momentum-non-iid-effect.png --ymax 100 --linestyles 'solid' 'dashed' 'dotted'
\begin{subfigure}[b]{0.45\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/d-cliques-cifar10-momentum-non-iid-effect}
@@ -579,7 +579,7 @@ IID node with momentum.
\end{subfigure}
\hfill
% To regenerate figure, from results/cifar10
- % python ../../../learn-topology/tools/plot_convergence.py 1-node-iid/all/2021-03-10-13:52:58-CET no-init/fully-connected-cliques/all/2021-03-13-18:32:55-CET --legend 'upper right' --add-min-max --labels '1-node IID w/ momentum' '100 nodes non-IID d-clique w/ momentum' --font-size 14 --yaxis test-accuracy --save-figure ../../figures/d-cliques-cifar10-momentum-non-iid-clique-avg-effect.png --ymax 100
+ % python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py 1-node-iid/all/2021-03-10-13:52:58-CET no-init/fully-connected-cliques/all/2021-03-13-18:32:55-CET --legend 'upper right' --add-min-max --labels '1-node IID w/ momentum' '100 nodes non-IID d-clique w/ momentum' --font-size 14 --yaxis test-accuracy --save-figure ../../figures/d-cliques-cifar10-momentum-non-iid-clique-avg-effect.png --ymax 100 --linestyles 'solid' 'dashed' 'dotted'
\begin{subfigure}[b]{0.45\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/d-cliques-cifar10-momentum-non-iid-clique-avg-effect}
@@ -648,14 +648,14 @@ significantly reduces the variance across nodes in this setup.
\centering
\begin{subfigure}[b]{0.48\textwidth}
% To regenerate the figure, from directory results/mnist
-% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py fully-connected-cliques/all/2021-03-10-10:19:44-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-12-11:12:49-CET random-10/all/2021-07-23-11:59:56-CEST random-10-diverse/all/2021-03-17-20:28:35-CET --labels 'd-clique (fcc)' 'd-clique (fcc) no clique avg.' '10 random edges' '10 random edges (all classes represented)' --add-min-max --legend 'lower right' --ymin 80 --ymax 92.5 --yaxis test-accuracy --save-figure ../../figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png --font-size 13
+% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py fully-connected-cliques/all/2021-03-10-10:19:44-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-12-11:12:49-CET random-10/all/2021-07-23-11:59:56-CEST random-10-diverse/all/2021-03-17-20:28:35-CET --labels 'd-clique (fcc)' 'd-clique (fcc) no clique avg.' '10 random edges' '10 random edges (all classes represented)' --add-min-max --legend 'lower right' --ymin 80 --ymax 92.5 --yaxis test-accuracy --save-figure ../../figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png --font-size 13 --linestyles 'solid' 'dashed' 'dotted' 'dashdot'
\centering
\includegraphics[width=\textwidth]{figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies}
\caption{MNIST with Linear Model}
\end{subfigure}
\hfill
% To regenerate the figure, from directory results/cifar10
-% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py no-init/fully-connected-cliques/all/2021-03-13-18:32:55-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-13-18:34:35-CET random-10/all/2021-07-23-14:33:48-CEST random-10-diverse/all/2021-03-17-20:30:41-CET random-10-diverse-unbiased-gradient/all/2021-03-17-20:31:14-CET --labels 'd-clique (fcc) clique avg.' 'd-clique (fcc) no clique avg.' '10 random edges' '10 random edges (all classes repr.)' '10 random (all classes repr.) with unbiased grad.' --add-min-max --legend 'upper left' --yaxis test-accuracy --save-figure ../../figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png --ymax 119 --font-size 13
+% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py no-init/fully-connected-cliques/all/2021-03-13-18:32:55-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-13-18:34:35-CET random-10/all/2021-07-23-14:33:48-CEST random-10-diverse/all/2021-03-17-20:30:41-CET random-10-diverse-unbiased-gradient/all/2021-03-17-20:31:14-CET --labels 'd-clique (fcc) clique avg.' 'd-clique (fcc) no clique avg.' '10 random edges' '10 random edges (all classes repr.)' '10 random (all classes repr.) with unbiased grad.' --add-min-max --legend 'upper left' --yaxis test-accuracy --save-figure ../../figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png --ymax 119 --font-size 13 --linestyles 'solid' 'dashed' 'dotted' 'dashdot' 'solid' --markers '' '' '' '' 'o'
\begin{subfigure}[b]{0.48\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies}
@@ -679,7 +679,7 @@ instrumental in obtaining good performance, one may wonder whether
intra-clique full connectivity is actually necessary.
Figure~\ref{fig:d-cliques-intra-connectivity} shows the convergence speed of
a D-Cliques topology where cliques have been sparsified by randomly
-removing 1 or 5 edges per clique (out of 45). Strikingly, both for MNIST and
+removing 1 or 5 undirected edges per clique (out of 45). Strikingly, both for MNIST and
CIFAR10, removing just a single edge from the cliques has a
significant effect on the
convergence speed. On CIFAR10, it even entirely negates the
@@ -695,17 +695,33 @@ proposed with D-Cliques.
\begin{subfigure}[htbp]{0.48\textwidth}
\centering
% To regenerate the figure, from directory results/mnist
-% python ../../../learn-topology/tools/plot_convergence.py no-init/fully-connected-cliques/all/2021-03-12-11:12:01-CET rm-1-edge/all/2021-03-18-17:28:27-CET rm-5-edges/all/2021-03-18-17:29:10-CET rm-1-edge-unbiased-grad/all/2021-03-18-17:28:47-CET rm-5-edges-unbiased-grad/all/2021-03-18-17:29:36-CET --add-min-max --ymin 85 --ymax 92.5 --legend 'lower right' --yaxis test-accuracy --labels 'fcc with clique grad.' 'fcc -1 edge/clique, no clique grad.' 'fcc -5 edges/clique, no clique grad.' 'fcc -1 edge/clique, clique grad.' 'fcc -5 edges/clique, clique grad.' --save-figure ../../figures/d-cliques-mnist-clique-clustering-fcc.png --font-size 13
- \includegraphics[width=\textwidth]{figures/d-cliques-mnist-clique-clustering-fcc}
-\caption{\label{fig:d-cliques-mnist-clique-clustering} MNIST}
+% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py no-init/fully-connected-cliques/all/2021-03-12-11:12:01-CET rm-1-edge/all/2021-03-18-17:28:27-CET rm-1-edge-unbiased-grad/all/2021-03-18-17:28:47-CET --add-min-max --ymin 85 --ymax 92.5 --legend 'lower right' --yaxis test-accuracy --labels 'fcc, clique grad.' 'fcc -1 edge/clique, no clique grad.' 'fcc -1 edge/clique, clique grad.' --save-figure ../../figures/d-cliques-mnist-clique-clustering-fcc-minus-1-edge.png --font-size 13 --linestyle 'solid' 'dashed' 'dotted'
+ \includegraphics[width=\textwidth]{figures/d-cliques-mnist-clique-clustering-fcc-minus-1-edge}
+\caption{\label{fig:d-cliques-mnist-clique-clustering-minus-1-edge} MNIST (-1 edge/clique)}
\end{subfigure}
\hfill
\begin{subfigure}[htbp]{0.48\textwidth}
\centering
% To regenerate the figure, from directory results/cifar10
-% python ../../../learn-topology/tools/plot_convergence.py no-init/fully-connected-cliques/all/2021-03-13-18:32:55-CET rm-1-edge/all/2021-03-18-17:29:58-CET rm-5-edges/all/2021-03-18-17:30:38-CET rm-1-edge-unbiased-grad/all/2021-03-18-17:30:17-CET rm-5-edges-unbiased-grad/all/2021-03-18-17:31:04-CET --add-min-max --ymax 80 --legend 'upper left' --yaxis test-accuracy --labels 'fcc, clique grad.' 'fcc -1 edge/clique, no clique grad.' 'fcc -5 edges/clique, no clique grad.' 'fcc -1 edge/clique, clique grad.' 'fcc -5 edges/clique, clique grad.' --save-figure ../../figures/d-cliques-cifar10-clique-clustering-fcc.png --font-size 13
- \includegraphics[width=\textwidth]{figures/d-cliques-cifar10-clique-clustering-fcc}
-\caption{\label{fig:d-cliques-cifar10-clique-clustering} CIFAR10}
+% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py no-init/fully-connected-cliques/all/2021-03-13-18:32:55-CET rm-1-edge/all/2021-03-18-17:29:58-CET rm-1-edge-unbiased-grad/all/2021-03-18-17:30:17-CET --add-min-max --ymax 80 --legend 'upper left' --yaxis test-accuracy --labels 'fcc, clique grad.' 'fcc -1 edge/clique, no clique grad.' 'fcc -1 edge/clique, clique grad.' --save-figure ../../figures/d-cliques-cifar10-clique-clustering-fcc-minus-1-edge.png --font-size 13 --linestyle 'solid' 'dashed' 'dotted'
+ \includegraphics[width=\textwidth]{figures/d-cliques-cifar10-clique-clustering-fcc-minus-1-edge}
+\caption{\label{fig:d-cliques-cifar10-clique-clustering-minus-1-edge} CIFAR10 (-1 edge/clique)}
+\end{subfigure}
+
+\begin{subfigure}[htbp]{0.48\textwidth}
+ \centering
+% To regenerate the figure, from directory results/mnist
+% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py no-init/fully-connected-cliques/all/2021-03-12-11:12:01-CET rm-5-edges/all/2021-03-18-17:29:10-CET rm-5-edges-unbiased-grad/all/2021-03-18-17:29:36-CET --add-min-max --ymin 85 --ymax 92.5 --legend 'lower right' --yaxis test-accuracy --labels 'fcc, clique grad.' 'fcc -5 edges/clique, no clique grad.' 'fcc -5 edges/clique, clique grad.' --save-figure ../../figures/d-cliques-mnist-clique-clustering-fcc-minus-5-edges.png --font-size 13 --linestyle 'solid' 'dashed' 'dotted'
+ \includegraphics[width=\textwidth]{figures/d-cliques-mnist-clique-clustering-fcc-minus-5-edges}
+\caption{\label{fig:d-cliques-mnist-clique-clustering-minus-5-edges} MNIST (-5 edges/clique)}
+\end{subfigure}
+\hfill
+\begin{subfigure}[htbp]{0.48\textwidth}
+ \centering
+% To regenerate the figure, from directory results/cifar10
+% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py no-init/fully-connected-cliques/all/2021-03-13-18:32:55-CET rm-5-edges/all/2021-03-18-17:30:38-CET rm-5-edges-unbiased-grad/all/2021-03-18-17:31:04-CET --add-min-max --ymax 80 --legend 'upper left' --yaxis test-accuracy --labels 'fcc, clique grad.' 'fcc -5 edges/clique, no clique grad.' 'fcc -5 edges/clique, clique grad.' --save-figure ../../figures/d-cliques-cifar10-clique-clustering-fcc-minus-5-edges.png --font-size 13 --linestyle 'solid' 'dashed' 'dotted'
+ \includegraphics[width=\textwidth]{figures/d-cliques-cifar10-clique-clustering-fcc-minus-5-edges}
+\caption{\label{fig:d-cliques-cifar10-clique-clustering-minus-5-edges} CIFAR10 (-5 edges/clique)}
\end{subfigure}
\caption{\label{fig:d-cliques-intra-connectivity} Importance of Intra-Clique Full-Connectivity}
@@ -717,7 +733,7 @@ proposed with D-Cliques.
So far, we have used a fully-connected inter-clique topology for D-Cliques,
which has the advantage of bounding the
-average shortest path to $3$ between any pair of nodes. This choice requires $
+\textit{path length}\footnote{The \textit{path length} is the number of edges on the path with the shortest number of edges between two nodes.} to $3$ between any pair of nodes. This choice requires $
\frac{n}{c}(\frac{n}{c} - 1)$ inter-clique edges, which scales quadratically
in the number of nodes $n$ for a given clique size $c$\footnote{We consider \textit{directed} edges in the analysis: the number of undirected edges is half and does not affect asymptotic behavior.}. This can become significant at larger scales when $n$ is
large compared to $c$.
@@ -728,17 +744,13 @@ nodes. We compare the scalability and convergence speed of several
D-Cliques variants, which all use $O(nc)$ edges
to create cliques as a starting point.
-The inter-clique topology with (almost) fewest possible edges is a
-\textit{ring}, which
-uses $\frac{2n}{c}$ inter-clique edges and therefore scales linearly in
-$n$.
-We also consider another topology that scales linearly and achieves a
+We first measure the convergence speed of inter-cliques topologies whose number of edges scales linearly with the number of nodes. Among those, the \textit{ring} has the (almost) fewest possible number of edges: it
+uses $\frac{2n}{c}$ inter-clique edges but its average path length between nodes
+also scales linearly.
+We also consider another topology, which we call \textit{fractal}, that provides a
logarithmic
-bound on the
-average shortest number of hops between two nodes. In this hierarchical scheme
-that we call \textit{fractal}, cliques are
-assembled in
-larger groups of $c$ cliques that are connected internally with one edge per
+bound on the average path length. In this hierarchical scheme,
+cliques are assembled in larger groups of $c$ cliques that are connected internally with one edge per
pair of cliques, but with only one edge between pairs of larger groups. The
topology is built recursively such that $c$ groups will themselves form a
larger group at the next level up. This results in at most $c$ edges per node
@@ -746,9 +758,11 @@ if edges are evenly distributed: i.e., each group within the same level adds
at most $c-1$ edges to other groups, leaving one node per group with $c-1$
edges that can receive an additional edge to connect with other groups at the next level.
Since nodes have at most $c$ edges, $n$ nodes have at most $nc$ edges, therefore
-this fractal scheme also scales linearly in the number of nodes.
+the number of edges in this fractal scheme indeed scales linearly in the number of nodes.
-Finally, we propose to connect cliques according to a
+Second, we look at another scheme
+in which the number of edges scales in a near, but not quite, linear fashion.
+We propose to connect cliques according to a
small-world-like topology~\cite{watts2000small} applied on top of a
ring~\cite{stoica2003chord}. In this scheme, cliques are first arranged in a
ring. Then each clique adds symmetric edges, both clockwise and
@@ -756,8 +770,8 @@ counter-clockwise on the ring, with the $m$ closest cliques in sets of
cliques that are exponentially bigger the further they are on the ring (see
Algorithm~\ref{Algorithm:Smallworld} in the appendix for
details on the construction). This ensures a good connectivity with other
-cliques that are close on the ring, while still keeping the average shortest
-path small. This scheme uses $\frac{n}{c}*2(m)\log(\frac{n}{c})$ inter-clique edges and
+cliques that are close on the ring, while still keeping the average
+path length small. This scheme uses $\frac{n}{c}*2(m)\log(\frac{n}{c})$ inter-clique edges and
therefore grows in the order of $O(n\log(n))$ with the number of nodes.
Figure~\ref{fig:d-cliques-cifar10-convolutional} shows the convergence
@@ -765,14 +779,14 @@ speed of all the above schemes on MNIST and CIFAR10, compared to the ideal
baseline
of a
single IID node performing the same number of updates per epoch (representing
-the fastest convergence speed achievable if topology had no impact). The ring
-topology converges but is much slower, while our fractal scheme helps
-significantly. The sweet spot appears to be the small-world
-topology, as the convergence speed is almost the same as with a
+the fastest convergence speed achievable if topology had no impact). Among the linear schemes, the ring
+topology converges but is much slower than our fractal scheme. Among the super-linear schemes, the small-world
+topology has a convergence speed that is almost the same as with a
fully-connected inter-clique topology but with 22\% less edges
-(14.5 edges on average instead of 18.9). Note that we can expect bigger
-gains at larger scales. Nonetheless, we stress the fact that even the
-fully-connected topology offers
+(14.5 edges on average instead of 18.9).
+
+While the small-world inter-clique topology shows promising scaling behaviour, the
+fully-connected topology still offers
significant benefits with 1000 nodes, as it represents a 98\% reduction in the
number of edges compared to fully connecting individual nodes (18.9 edges on
average instead of 999) and a 96\% reduction in the number of messages (37.8
@@ -783,20 +797,37 @@ show that D-Cliques can nicely scale with the number of nodes.
\begin{figure}[t]
\centering
+ % To regenerate the figure, from directory results/mnist
+ % python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py 1-node-iid/all/2021-03-10-09:20:03-CET ../scaling/1000/mnist/fractal-cliques/all/2021-03-14-17:41:59-CET ../scaling/1000/mnist/clique-ring/all/2021-03-13-18:22:36-CET --add-min-max --yaxis test-accuracy --legend 'lower right' --ymin 84 --ymax 92.5 --labels '1 node IID' 'd-cliques (fractal)' 'd-cliques (ring)' --save-figure ../../figures/d-cliques-mnist-1000-nodes-comparison-linear.png --font-size 13 --linestyles 'solid' 'dashed' 'dotted'
+ \begin{subfigure}[b]{0.48\textwidth}
+ \centering
+ \includegraphics[width=\textwidth]{figures/d-cliques-mnist-1000-nodes-comparison-linear}
+ \caption{\label{fig:d-cliques-mnist-1000-nodes-comparison-linear} MNIST with Linear Model: Linear Inter-clique Topologies.}
+ \end{subfigure}
+ \hfill
+ % To regenerate the figure, from directory results/cifar10
+% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py 1-node-iid/all/2021-03-10-13:52:58-CET ../scaling/1000/cifar10/fractal-cliques/all/2021-03-14-17:42:46-CET ../scaling/1000/cifar10/clique-ring/all/2021-03-14-09:55:24-CET --add-min-max --yaxis test-accuracy --labels '1-node IID' 'd-cliques (fractal)' 'd-cliques (ring)' --legend 'lower right' --save-figure ../../figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-linear.png --font-size 13 --linestyles 'solid' 'dashed' 'dotted'
+ \begin{subfigure}[b]{0.48\textwidth}
+ \centering
+ \includegraphics[width=\textwidth]{figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-linear}
+\caption{\label{fig:d-cliques-cifar10-1000-vs-1-node-test-accuracy-linear} CIFAR10 with LeNet Model: Linear Inter-clique Topologies.}
+ \end{subfigure}
+
+
% To regenerate the figure, from directory results/mnist
- % python ../../../learn-topology/tools/plot_convergence.py 1-node-iid/all/2021-03-10-09:20:03-CET ../scaling/1000/mnist/fully-connected-cliques/all/2021-03-14-17:56:26-CET ../scaling/1000/mnist/smallworld-logn-cliques/all/2021-03-23-21:45:39-CET ../scaling/1000/mnist/fractal-cliques/all/2021-03-14-17:41:59-CET ../scaling/1000/mnist/clique-ring/all/2021-03-13-18:22:36-CET --add-min-max --yaxis test-accuracy --legend 'lower right' --ymin 84 --ymax 92.5 --labels '1 node IID' 'd-cliques (fully-connected cliques)' 'd-cliques (smallworld)' 'd-cliques (fractal)' 'd-cliques (ring)' --save-figure ../../figures/d-cliques-mnist-1000-nodes-comparison.png --font-size 13
+ % python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py 1-node-iid/all/2021-03-10-09:20:03-CET ../scaling/1000/mnist/fully-connected-cliques/all/2021-03-14-17:56:26-CET ../scaling/1000/mnist/smallworld-logn-cliques/all/2021-03-23-21:45:39-CET --add-min-max --yaxis test-accuracy --legend 'lower right' --ymin 84 --ymax 92.5 --labels '1 node IID' 'd-cliques (fully-connected cliques)' 'd-cliques (smallworld)' --save-figure ../../figures/d-cliques-mnist-1000-nodes-comparison-super-linear.png --font-size 13 --linestyles 'solid' 'dashed' 'dotted'
\begin{subfigure}[b]{0.48\textwidth}
\centering
- \includegraphics[width=\textwidth]{figures/d-cliques-mnist-1000-nodes-comparison}
- \caption{\label{fig:d-cliques-mnist-1000-nodes-comparison} MNIST with Linear}
+ \includegraphics[width=\textwidth]{figures/d-cliques-mnist-1000-nodes-comparison-super-linear}
+ \caption{\label{fig:d-cliques-mnist-1000-nodes-comparison-super-linear} MNIST with Linear Model: Superlinear Inter-clique Topologies.}
\end{subfigure}
\hfill
% To regenerate the figure, from directory results/cifar10
-% python ../../../learn-topology/tools/plot_convergence.py 1-node-iid/all/2021-03-10-13:52:58-CET ../scaling/1000/cifar10/fully-connected-cliques/all/2021-03-14-17:41:20-CET ../scaling/1000/cifar10/smallworld-logn-cliques/all/2021-03-23-22:13:57-CET ../scaling/1000/cifar10/fractal-cliques/all/2021-03-14-17:42:46-CET ../scaling/1000/cifar10/clique-ring/all/2021-03-14-09:55:24-CET --add-min-max --yaxis test-accuracy --labels '1-node IID' 'd-cliques (fully-connected cliques)' 'd-cliques (smallworld)' 'd-cliques (fractal)' 'd-cliques (ring)' --legend 'lower right' --save-figure ../../figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy.png --font-size 13
+% python ../../../../Software/non-iid-topology-simulator/tools/plot_convergence.py 1-node-iid/all/2021-03-10-13:52:58-CET ../scaling/1000/cifar10/fully-connected-cliques/all/2021-03-14-17:41:20-CET ../scaling/1000/cifar10/smallworld-logn-cliques/all/2021-03-23-22:13:57-CET --add-min-max --yaxis test-accuracy --labels '1-node IID' 'd-cliques (fully-connected cliques)' 'd-cliques (smallworld)' --legend 'lower right' --save-figure ../../figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-super-linear.png --font-size 13 --linestyles 'solid' 'dashed' 'dotted'
\begin{subfigure}[b]{0.48\textwidth}
\centering
- \includegraphics[width=\textwidth]{figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy}
-\caption{\label{fig:d-cliques-cifar10-1000-vs-1-node-test-accuracy} CIFAR10 with LeNet}
+ \includegraphics[width=\textwidth]{figures/d-cliques-cifar10-1000-vs-1-node-test-accuracy-super-linear}
+\caption{\label{fig:d-cliques-cifar10-1000-vs-1-node-test-accuracy-super-linear} CIFAR10 with LeNet Model: Superlinear Inter-clique Topologies.}
\end{subfigure}
\caption{\label{fig:d-cliques-cifar10-convolutional} D-Cliques Convergence Speed with 1000 nodes, non-IID, Constant Updates per Epoch, with Different Inter-Clique Topologies.}