diff --git a/figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png b/figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png
new file mode 100644
index 0000000000000000000000000000000000000000..6ff1a68cdd294c266ec8c88d94cd583368abcf1f
Binary files /dev/null and b/figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png differ
diff --git a/figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png b/figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png
new file mode 100644
index 0000000000000000000000000000000000000000..7c458bc0ed03c7e6dc0212381990f44c5da8192d
Binary files /dev/null and b/figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png differ
diff --git a/main.tex b/main.tex
index 9dbddad363240121706701aad2442dd51093882b..c85ce04997c31bad038d7628abaa7efa75213d8c 100644
--- a/main.tex
+++ b/main.tex
@@ -284,21 +284,6 @@ We solve this problem by decoupling the gradient averaging from the weight avera
\caption{\label{fig:d-cliques-mnist-init-clique-avg-effect-fcc-test-accuracy} Fully-Connected}
\end{subfigure}
\caption{\label{fig:d-cliques-mnist-init-clique-avg-effect} MNIST: Effects of Clique Averaging and Uniform Initialization on Convergence Speed. (100 nodes, non-IID, D-Cliques, bsz=128)}
-\end{figure}
-
- \begin{figure}[htbp]
- \centering
- \begin{subfigure}[b]{0.48\textwidth}
- \centering
- %\includegraphics[width=0.7\textwidth]{figures/d-cliques-mnist-vs-1-node-test-accuracy}
- \caption{\label{fig:mnist-clique-clustering-ring} Ring}
- \end{subfigure}
- \begin{subfigure}[b]{0.48\textwidth}
- \centering
- %\includegraphics[width=0.7\textwidth]{figures/d-cliques-mnist-vs-1-node-test-accuracy}
- \caption{\label{fig:mnist-clique-clustering-fcc} Fully-Connected}
- \end{subfigure}
-\caption{\label{fig:d-cliques-mnist-clique-clustering} MNIST: Effects of Clustering within Cliques on Convergence Speed.}
\end{figure}
% REMOVED: Constant Batch-size
@@ -341,9 +326,33 @@ We solve this problem by decoupling the gradient averaging from the weight avera
\caption{\label{fig:d-cliques-mnist-scaling-fully-connected} MNIST: D-Clique Scaling Behaviour, where $n$ is the number of nodes, and $c$ the size of a clique (Constant Updates per Epoch).}
\end{figure}
-Show scaling effect for 10, 100, 1000 nodes (with decreasing sample sizes) for Clique Ring, Fractal, Fully-Connected.
-Robustness to not having fully-connected cliques (static and dynamic subsets).
+
+ \begin{figure}[htbp]
+ \centering
+
+% To regenerate the figure, from directory results/mnist
+% python ../../../learn-topology/tools/plot_convergence.py fully-connected-cliques/all/2021-03-10-10:19:44-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-12-11:12:49-CET random-10/all/2021-03-17-20:28:12-CET random-10-diverse/all/2021-03-17-20:28:35-CET --labels 'd-clique (fcc)' 'd-clique (fcc) no clique avg. no uniform init.' '10 random edges' '10 random edges (all classes represented)' --add-min-max --legend 'lower right' --ymin 88 --ymax 92.5 --yaxis test-accuracy --save-figure ../../figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies.png
+ \centering
+ \includegraphics[width=0.6\textwidth]{figures/d-cliques-mnist-linear-comparison-to-non-clustered-topologies}
+ \caption{\label{fig:d-cliques-mnist-linear-comparison-to-non-clustered-topologies} MNIST: Comparison to non-Clustered Topologies}
+ \end{figure}
+
+
+ \begin{figure}[htbp]
+ \centering
+ \begin{subfigure}[b]{0.48\textwidth}
+ \centering
+ %\includegraphics[width=0.7\textwidth]{figures/d-cliques-mnist-vs-1-node-test-accuracy}
+ \caption{\label{fig:mnist-clique-clustering-ring} Ring}
+ \end{subfigure}
+ \begin{subfigure}[b]{0.48\textwidth}
+ \centering
+ %\includegraphics[width=0.7\textwidth]{figures/d-cliques-mnist-vs-1-node-test-accuracy}
+ \caption{\label{fig:mnist-clique-clustering-fcc} Fully-Connected}
+ \end{subfigure}
+\caption{\label{fig:d-cliques-mnist-clique-clustering} MNIST: Effect of Relaxed Intra-Clique Connectivity.}
+\end{figure}
\clearpage
@@ -404,22 +413,6 @@ In addition, it is important that all nodes are initialized with the same model
\caption{\label{fig:d-cliques-cifar10-init-clique-avg-effect} CIFAR10: Effects of Clique Averaging and Uniform Initialization on Convergence Speed. (100 nodes, non-IID, D-Cliques, bsz=20)}
\end{figure}
- \begin{figure}[htbp]
- \centering
- \begin{subfigure}[b]{0.48\textwidth}
- \centering
- %\includegraphics[width=0.7\textwidth]{figures/d-cliques-cifar10-vs-1-node-test-accuracy}
- \caption{\label{fig:cifar10-clique-clustering-ring} Ring}
- \end{subfigure}
- \begin{subfigure}[b]{0.48\textwidth}
- \centering
- %\includegraphics[width=0.7\textwidth]{figures/d-cliques-cifar10-vs-1-node-test-accuracy}
- \caption{\label{fig:cifar10-clique-clustering-fcc} Fully-Connected}
- \end{subfigure}
-\caption{\label{fig:d-cliques-cifar10-clique-clustering} CIFAR10: Effects of Clustering within Cliques on Convergence Speed.}
-\end{figure}
-
-
\begin{figure}[htbp]
\centering
@@ -453,21 +446,30 @@ In addition, it is important that all nodes are initialized with the same model
\caption{\label{fig:d-cliques-cifar10-scaling-fully-connected} CIFAR10: D-Clique Scaling Behaviour, where $n$ is the number of nodes, and $c$ the size of a clique (Constant Updates per Epoch).}
\end{figure}
+ \begin{figure}[htbp]
+ \centering
+% To regenerate the figure, from directory results/cifar10
+% python ../../../learn-topology/tools/plot_convergence.py fully-connected-cliques/all/2021-03-10-13:58:57-CET no-init-no-clique-avg/fully-connected-cliques/all/2021-03-13-18:34:35-CET random-10/all/2021-03-17-20:30:03-CET random-10-diverse/all/2021-03-17-20:30:41-CET random-10-diverse-unbiased-gradient/all/2021-03-17-20:31:14-CET random-10-diverse-unbiased-gradient-uniform-init/all/2021-03-17-20:31:41-CET --labels 'd-clique (fcc) clique avg., uniform init.' 'd-clique (fcc) no clique avg. no uniform init.' '10 random edges' '10 random edges (all classes repr.)' '10 random (all classes repr.) with unbiased grad.' '10 random (all classes repr.) with unbiased grad., uniform init.' --add-min-max --legend 'upper left' --yaxis test-accuracy --save-figure ../../figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies.png --ymax 100
+ \includegraphics[width=0.6\textwidth]{figures/d-cliques-cifar10-linear-comparison-to-non-clustered-topologies}
+ \caption{\label{fig:d-cliques-cifar10-linear-comparison-to-non-clustered-topologies} CIFAR10: Comparison to non-Clustered Topologies}
+ \end{figure}
-\subsection{Comparison to similar topologies}
-
-Similar number of maximum hops but no or less clustering than D-Cliques (and no unbiasing of gradient).
+ \begin{figure}[htbp]
+ \centering
+ \begin{subfigure}[b]{0.48\textwidth}
+ \centering
+ %\includegraphics[width=0.7\textwidth]{figures/d-cliques-cifar10-vs-1-node-test-accuracy}
+ \caption{\label{fig:cifar10-clique-clustering-ring} Ring}
+ \end{subfigure}
+ \begin{subfigure}[b]{0.48\textwidth}
+ \centering
+ %\includegraphics[width=0.7\textwidth]{figures/d-cliques-cifar10-vs-1-node-test-accuracy}
+ \caption{\label{fig:cifar10-clique-clustering-fcc} Fully-Connected}
+ \end{subfigure}
+\caption{\label{fig:d-cliques-cifar10-clique-clustering} CIFAR10: Effect of Relaxed Intra-Clique Connectivity.}
+\end{figure}
-\begin{itemize}
- \item Choice of 10 random neighbours (static) in a fully connected graph (D-PSGD aléatoire), such that all nodes have at most 10 edges
- \item 10 random neighbours, such that all nodes have at most 10 edges, but all nodes have neighbours of all classes
- \item item previous, with neighbour averaging
- \item item previous, with neighbour averaging and uniform initialization
- %\item Uniform Diverse Neighbourhood with No Clustering
- %\item Random network
- %\item Random Small-World Graph
-\end{itemize}
\section{Related Work}