diff --git a/eval/epoch_configs/config_celeba.ini b/eval/epoch_configs/config_celeba.ini
index 782de8ac9d7c9dae2821169125df8aea633eaa05..0a6e7159e0feace34be8cbb9fbdde8ea05c940fe 100644
--- a/eval/epoch_configs/config_celeba.ini
+++ b/eval/epoch_configs/config_celeba.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/epoch_configs/config_celeba_100.ini b/eval/epoch_configs/config_celeba_100.ini
index 3a8ea4727e906521cf4dd833ad971ff7396d727b..8fed4d6c9d6bd598fb356a7b2016d2e8b302d0d0 100644
--- a/eval/epoch_configs/config_celeba_100.ini
+++ b/eval/epoch_configs/config_celeba_100.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/epoch_configs/config_celeba_grow.ini b/eval/epoch_configs/config_celeba_grow.ini
index 8f0b7180a02deb8bd3c17176915ee03eedcf8dae..9c2dfef3972a64fb29f26c5d25ae2bfcde71015e 100644
--- a/eval/epoch_configs/config_celeba_grow.ini
+++ b/eval/epoch_configs/config_celeba_grow.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/epoch_configs/config_femnist.ini b/eval/epoch_configs/config_femnist.ini
index b16bd9aa25f9302fd16e0f9e7117d1b5ba273390..13569c1106073e419e4686c635d613407671db44 100644
--- a/eval/epoch_configs/config_femnist.ini
+++ b/eval/epoch_configs/config_femnist.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/epoch_configs/config_femnist_100.ini b/eval/epoch_configs/config_femnist_100.ini
index 0f7dd023b31b6556818962fb9c57fec6fa37ad56..4e05952966d60bc4e60793e01964b575d75765fb 100644
--- a/eval/epoch_configs/config_femnist_100.ini
+++ b/eval/epoch_configs/config_femnist_100.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/epoch_configs/config_femnist_grow.ini b/eval/epoch_configs/config_femnist_grow.ini
index b240c8410c86266f3f8388cdcaea805ad5535891..75fb9d1efd9c28288a68cb53a743bbc7985723f7 100644
--- a/eval/epoch_configs/config_femnist_grow.ini
+++ b/eval/epoch_configs/config_femnist_grow.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/step_configs/config_celeba.ini b/eval/step_configs/config_celeba.ini
index d58052e5cc37725c68cd04fb8b8723f1859b8917..5cadf017749dbb602188323cbca82a1dfd40dcac 100644
--- a/eval/step_configs/config_celeba.ini
+++ b/eval/step_configs/config_celeba.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/step_configs/config_celeba_100.ini b/eval/step_configs/config_celeba_100.ini
index 2e351a47bcb148001c3895049b7f2d14458a43f0..70e14bbaf2fe87717351567aa4420ce439e3fa98 100644
--- a/eval/step_configs/config_celeba_100.ini
+++ b/eval/step_configs/config_celeba_100.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/step_configs/config_celeba_grow.ini b/eval/step_configs/config_celeba_grow.ini
index a9cd92c46417dce619eb2fab64f3552722e5d8d4..be0812e8c3bc1b91d14d803c3efe89e6d8e9b3e8 100644
--- a/eval/step_configs/config_celeba_grow.ini
+++ b/eval/step_configs/config_celeba_grow.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/step_configs/config_femnist.ini b/eval/step_configs/config_femnist.ini
index 0bd4a55dbfd136bf70d5a1fe5ba23f3ca3d3d260..43bb07dc32e862ed73155ba8a879df569d8266c3 100644
--- a/eval/step_configs/config_femnist.ini
+++ b/eval/step_configs/config_femnist.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/step_configs/config_femnist_100.ini b/eval/step_configs/config_femnist_100.ini
index becf623fe8e80f9ce8eeb267b9bb79685a76684e..4e3e9ba57519f265240130dc8c054355e3bd4d18 100644
--- a/eval/step_configs/config_femnist_100.ini
+++ b/eval/step_configs/config_femnist_100.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/step_configs/config_femnist_grow.ini b/eval/step_configs/config_femnist_grow.ini
index 018cdc76da7417340637703348d08dc2d3c9c3d8..9f18ad972e2c5b27e4ffca2242e08488290c083e 100644
--- a/eval/step_configs/config_femnist_grow.ini
+++ b/eval/step_configs/config_femnist_grow.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/src/decentralizepy/datasets/Celeba.py b/src/decentralizepy/datasets/Celeba.py
index 10b99cbbd2726fe488168aa1524964f307c5817c..61a77c61c3257c062398de10b73ce314300d0064 100644
--- a/src/decentralizepy/datasets/Celeba.py
+++ b/src/decentralizepy/datasets/Celeba.py
@@ -15,6 +15,7 @@ import decentralizepy.utils as utils
from decentralizepy.datasets.Data import Data
from decentralizepy.datasets.Dataset import Dataset
from decentralizepy.datasets.Partitioner import DataPartitioner
+from decentralizepy.mappings.Mapping import Mapping
from decentralizepy.models.Model import Model
IMAGE_DIM = 84
@@ -185,10 +186,9 @@ class Celeba(Dataset):
def __init__(
self,
- rank,
- machine_id,
- mapping,
- n_procs="",
+ rank: int,
+ machine_id: int,
+ mapping: Mapping,
train_dir="",
test_dir="",
images_dir="",
@@ -205,12 +205,11 @@ class Celeba(Dataset):
machine_id : int
Machine ID
mapping : decentralizepy.mappings.Mapping
- Mapping to conver rank, machine_id -> uid for data partitioning
- n_procs : int, optional
- The number of processes among which to divide the data. Default value is assigned 1
+ Mapping to convert rank, machine_id -> uid for data partitioning
+ It also provides the total number of global processes
train_dir : str, optional
Path to the training data files. Required to instantiate the training set
- The training set is partitioned according to n_procs and sizes
+ The training set is partitioned according to the number of global processes and sizes
test_dir : str. optional
Path to the testing data files Required to instantiate the testing set
sizes : list(int), optional
@@ -224,7 +223,6 @@ class Celeba(Dataset):
rank,
machine_id,
mapping,
- n_procs,
train_dir,
test_dir,
sizes,
diff --git a/src/decentralizepy/datasets/Dataset.py b/src/decentralizepy/datasets/Dataset.py
index 28c36e5af3f87ee9ebc89bdffabf5f0ddf819d1f..5d91f52e73da74183e5d6ed086f0ce33e8c6512f 100644
--- a/src/decentralizepy/datasets/Dataset.py
+++ b/src/decentralizepy/datasets/Dataset.py
@@ -1,5 +1,5 @@
from decentralizepy import utils
-
+from decentralizepy.mappings.Mapping import Mapping
class Dataset:
"""
@@ -10,10 +10,9 @@ class Dataset:
def __init__(
self,
- rank,
- machine_id,
- mapping,
- n_procs="",
+ rank: int,
+ machine_id: int,
+ mapping: Mapping,
train_dir="",
test_dir="",
sizes="",
@@ -29,12 +28,11 @@ class Dataset:
machine_id : int
Machine ID
mapping : decentralizepy.mappings.Mapping
- Mapping to conver rank, machine_id -> uid for data partitioning
- n_procs : int, optional
- The number of processes among which to divide the data. Default value is assigned 1
+ Mapping to convert rank, machine_id -> uid for data partitioning
+ It also provides the total number of global processes
train_dir : str, optional
Path to the training data files. Required to instantiate the training set
- The training set is partitioned according to n_procs and sizes
+ The training set is partitioned according to the number of global processes and sizes
test_dir : str. optional
Path to the testing data files Required to instantiate the testing set
sizes : list(int), optional
@@ -47,7 +45,8 @@ class Dataset:
self.rank = rank
self.machine_id = machine_id
self.mapping = mapping
- self.n_procs = utils.conditional_value(n_procs, "", 1)
+ # the number of global processes, needed to split-up the dataset
+ self.n_procs = mapping.get_n_procs()
self.train_dir = utils.conditional_value(train_dir, "", None)
self.test_dir = utils.conditional_value(test_dir, "", None)
self.sizes = utils.conditional_value(sizes, "", None)
diff --git a/src/decentralizepy/datasets/Femnist.py b/src/decentralizepy/datasets/Femnist.py
index 2e80096dec27833d1909b85e80840d84b35b53fb..23e9704f6d91b35e7f8ca2150b6d57b209d76ff7 100644
--- a/src/decentralizepy/datasets/Femnist.py
+++ b/src/decentralizepy/datasets/Femnist.py
@@ -13,6 +13,7 @@ from torch.utils.data import DataLoader
from decentralizepy.datasets.Data import Data
from decentralizepy.datasets.Dataset import Dataset
from decentralizepy.datasets.Partitioner import DataPartitioner
+from decentralizepy.mappings.Mapping import Mapping
from decentralizepy.models.Model import Model
NUM_CLASSES = 62
@@ -179,9 +180,9 @@ class Femnist(Dataset):
def __init__(
self,
- rank,
- machine_id,
- mapping,
+ rank: int,
+ machine_id: int,
+ mapping: Mapping,
n_procs="",
train_dir="",
test_dir="",
@@ -198,12 +199,11 @@ class Femnist(Dataset):
machine_id : int
Machine ID
mapping : decentralizepy.mappings.Mapping
- Mapping to conver rank, machine_id -> uid for data partitioning
- n_procs : int, optional
- The number of processes among which to divide the data. Default value is assigned 1
+ Mapping to convert rank, machine_id -> uid for data partitioning
+ It also provides the total number of global processes
train_dir : str, optional
Path to the training data files. Required to instantiate the training set
- The training set is partitioned according to n_procs and sizes
+ The training set is partitioned according to the number of global processes and sizes
test_dir : str. optional
Path to the testing data files Required to instantiate the testing set
sizes : list(int), optional
@@ -217,7 +217,6 @@ class Femnist(Dataset):
rank,
machine_id,
mapping,
- n_procs,
train_dir,
test_dir,
sizes,
diff --git a/src/decentralizepy/mappings/Mapping.py b/src/decentralizepy/mappings/Mapping.py
index b979eb96a753b18abf4b3ecfe9ee62561ad08b2d..9f764ed2875abd2dbaaf01e769c1c4d88d7d858b 100644
--- a/src/decentralizepy/mappings/Mapping.py
+++ b/src/decentralizepy/mappings/Mapping.py
@@ -18,6 +18,18 @@ class Mapping:
"""
self.n_procs = n_procs
+ def get_n_procs(self):
+ """
+ Gives the global sum of all processes that are spawned on the machines
+
+ Returns
+ -------
+ int
+ the number of global processes
+ """
+
+ return self.n_procs
+
def get_uid(self, rank: int, machine_id: int):
"""
Gives the global unique identifier of the node
diff --git a/src/decentralizepy/node/Node.py b/src/decentralizepy/node/Node.py
index e7aa9163818947d3fe523a3cf1f630aaa0f0ef41..0ef54cdee09002171d062f3b3f8ea2cf91e537f0 100644
--- a/src/decentralizepy/node/Node.py
+++ b/src/decentralizepy/node/Node.py
@@ -394,8 +394,6 @@ class Node:
Rank of process local to the machine
machine_id : int
Machine ID on which the process in running
- n_procs_local : int
- Number of processes on current machine
mapping : decentralizepy.mappings
The object containing the mapping rank <--> uid
graph : decentralizepy.graphs