From 2ee342af11eb6b7ee05fab2cc8057835200ef830 Mon Sep 17 00:00:00 2001
From: Jeffrey Wigger <jeffrey.wigger@epfl.ch>
Date: Thu, 17 Feb 2022 15:10:46 +0100
Subject: [PATCH] removing the n_proc config option and the code that replaces
it
---
eval/epoch_configs/config_celeba.ini | 1 -
eval/epoch_configs/config_celeba_100.ini | 1 -
eval/epoch_configs/config_celeba_grow.ini | 1 -
eval/epoch_configs/config_femnist.ini | 1 -
eval/epoch_configs/config_femnist_100.ini | 1 -
eval/epoch_configs/config_femnist_grow.ini | 1 -
eval/step_configs/config_celeba.ini | 1 -
eval/step_configs/config_celeba_100.ini | 1 -
eval/step_configs/config_celeba_grow.ini | 1 -
eval/step_configs/config_femnist.ini | 1 -
eval/step_configs/config_femnist_100.ini | 1 -
eval/step_configs/config_femnist_grow.ini | 1 -
src/decentralizepy/datasets/Celeba.py | 16 +++++++---------
src/decentralizepy/datasets/Dataset.py | 19 +++++++++----------
src/decentralizepy/datasets/Femnist.py | 15 +++++++--------
src/decentralizepy/mappings/Mapping.py | 12 ++++++++++++
src/decentralizepy/node/Node.py | 2 --
17 files changed, 35 insertions(+), 41 deletions(-)
diff --git a/eval/epoch_configs/config_celeba.ini b/eval/epoch_configs/config_celeba.ini
index 782de8a..0a6e715 100644
--- a/eval/epoch_configs/config_celeba.ini
+++ b/eval/epoch_configs/config_celeba.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/epoch_configs/config_celeba_100.ini b/eval/epoch_configs/config_celeba_100.ini
index 3a8ea47..8fed4d6 100644
--- a/eval/epoch_configs/config_celeba_100.ini
+++ b/eval/epoch_configs/config_celeba_100.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/epoch_configs/config_celeba_grow.ini b/eval/epoch_configs/config_celeba_grow.ini
index 8f0b718..9c2dfef 100644
--- a/eval/epoch_configs/config_celeba_grow.ini
+++ b/eval/epoch_configs/config_celeba_grow.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/epoch_configs/config_femnist.ini b/eval/epoch_configs/config_femnist.ini
index b16bd9a..13569c1 100644
--- a/eval/epoch_configs/config_femnist.ini
+++ b/eval/epoch_configs/config_femnist.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/epoch_configs/config_femnist_100.ini b/eval/epoch_configs/config_femnist_100.ini
index 0f7dd02..4e05952 100644
--- a/eval/epoch_configs/config_femnist_100.ini
+++ b/eval/epoch_configs/config_femnist_100.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/epoch_configs/config_femnist_grow.ini b/eval/epoch_configs/config_femnist_grow.ini
index b240c84..75fb9d1 100644
--- a/eval/epoch_configs/config_femnist_grow.ini
+++ b/eval/epoch_configs/config_femnist_grow.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/step_configs/config_celeba.ini b/eval/step_configs/config_celeba.ini
index d58052e..5cadf01 100644
--- a/eval/step_configs/config_celeba.ini
+++ b/eval/step_configs/config_celeba.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/step_configs/config_celeba_100.ini b/eval/step_configs/config_celeba_100.ini
index 2e351a4..70e14bb 100644
--- a/eval/step_configs/config_celeba_100.ini
+++ b/eval/step_configs/config_celeba_100.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/step_configs/config_celeba_grow.ini b/eval/step_configs/config_celeba_grow.ini
index a9cd92c..be0812e 100644
--- a/eval/step_configs/config_celeba_grow.ini
+++ b/eval/step_configs/config_celeba_grow.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Celeba
dataset_class = Celeba
model_class = CNN
-n_procs = 96
images_dir = /home/risharma/leaf/data/celeba/data/raw/img_align_celeba
train_dir = /home/risharma/leaf/data/celeba/per_user_data/train
test_dir = /home/risharma/leaf/data/celeba/data/test
diff --git a/eval/step_configs/config_femnist.ini b/eval/step_configs/config_femnist.ini
index 0bd4a55..43bb07d 100644
--- a/eval/step_configs/config_femnist.ini
+++ b/eval/step_configs/config_femnist.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/step_configs/config_femnist_100.ini b/eval/step_configs/config_femnist_100.ini
index becf623..4e3e9ba 100644
--- a/eval/step_configs/config_femnist_100.ini
+++ b/eval/step_configs/config_femnist_100.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/eval/step_configs/config_femnist_grow.ini b/eval/step_configs/config_femnist_grow.ini
index 018cdc7..9f18ad9 100644
--- a/eval/step_configs/config_femnist_grow.ini
+++ b/eval/step_configs/config_femnist_grow.ini
@@ -2,7 +2,6 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
-n_procs = 16
train_dir = /home/risharma/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/leaf/data/femnist/data/test
; python list of fractions below
diff --git a/src/decentralizepy/datasets/Celeba.py b/src/decentralizepy/datasets/Celeba.py
index 10b99cb..61a77c6 100644
--- a/src/decentralizepy/datasets/Celeba.py
+++ b/src/decentralizepy/datasets/Celeba.py
@@ -15,6 +15,7 @@ import decentralizepy.utils as utils
from decentralizepy.datasets.Data import Data
from decentralizepy.datasets.Dataset import Dataset
from decentralizepy.datasets.Partitioner import DataPartitioner
+from decentralizepy.mappings.Mapping import Mapping
from decentralizepy.models.Model import Model
IMAGE_DIM = 84
@@ -185,10 +186,9 @@ class Celeba(Dataset):
def __init__(
self,
- rank,
- machine_id,
- mapping,
- n_procs="",
+ rank: int,
+ machine_id: int,
+ mapping: Mapping,
train_dir="",
test_dir="",
images_dir="",
@@ -205,12 +205,11 @@ class Celeba(Dataset):
machine_id : int
Machine ID
mapping : decentralizepy.mappings.Mapping
- Mapping to conver rank, machine_id -> uid for data partitioning
- n_procs : int, optional
- The number of processes among which to divide the data. Default value is assigned 1
+ Mapping to convert rank, machine_id -> uid for data partitioning
+ It also provides the total number of global processes
train_dir : str, optional
Path to the training data files. Required to instantiate the training set
- The training set is partitioned according to n_procs and sizes
+ The training set is partitioned according to the number of global processes and sizes
test_dir : str. optional
Path to the testing data files Required to instantiate the testing set
sizes : list(int), optional
@@ -224,7 +223,6 @@ class Celeba(Dataset):
rank,
machine_id,
mapping,
- n_procs,
train_dir,
test_dir,
sizes,
diff --git a/src/decentralizepy/datasets/Dataset.py b/src/decentralizepy/datasets/Dataset.py
index 28c36e5..5d91f52 100644
--- a/src/decentralizepy/datasets/Dataset.py
+++ b/src/decentralizepy/datasets/Dataset.py
@@ -1,5 +1,5 @@
from decentralizepy import utils
-
+from decentralizepy.mappings.Mapping import Mapping
class Dataset:
"""
@@ -10,10 +10,9 @@ class Dataset:
def __init__(
self,
- rank,
- machine_id,
- mapping,
- n_procs="",
+ rank: int,
+ machine_id: int,
+ mapping: Mapping,
train_dir="",
test_dir="",
sizes="",
@@ -29,12 +28,11 @@ class Dataset:
machine_id : int
Machine ID
mapping : decentralizepy.mappings.Mapping
- Mapping to conver rank, machine_id -> uid for data partitioning
- n_procs : int, optional
- The number of processes among which to divide the data. Default value is assigned 1
+ Mapping to convert rank, machine_id -> uid for data partitioning
+ It also provides the total number of global processes
train_dir : str, optional
Path to the training data files. Required to instantiate the training set
- The training set is partitioned according to n_procs and sizes
+ The training set is partitioned according to the number of global processes and sizes
test_dir : str. optional
Path to the testing data files Required to instantiate the testing set
sizes : list(int), optional
@@ -47,7 +45,8 @@ class Dataset:
self.rank = rank
self.machine_id = machine_id
self.mapping = mapping
- self.n_procs = utils.conditional_value(n_procs, "", 1)
+ # the number of global processes, needed to split-up the dataset
+ self.n_procs = mapping.get_n_procs()
self.train_dir = utils.conditional_value(train_dir, "", None)
self.test_dir = utils.conditional_value(test_dir, "", None)
self.sizes = utils.conditional_value(sizes, "", None)
diff --git a/src/decentralizepy/datasets/Femnist.py b/src/decentralizepy/datasets/Femnist.py
index 2e80096..23e9704 100644
--- a/src/decentralizepy/datasets/Femnist.py
+++ b/src/decentralizepy/datasets/Femnist.py
@@ -13,6 +13,7 @@ from torch.utils.data import DataLoader
from decentralizepy.datasets.Data import Data
from decentralizepy.datasets.Dataset import Dataset
from decentralizepy.datasets.Partitioner import DataPartitioner
+from decentralizepy.mappings.Mapping import Mapping
from decentralizepy.models.Model import Model
NUM_CLASSES = 62
@@ -179,9 +180,9 @@ class Femnist(Dataset):
def __init__(
self,
- rank,
- machine_id,
- mapping,
+ rank: int,
+ machine_id: int,
+ mapping: Mapping,
n_procs="",
train_dir="",
test_dir="",
@@ -198,12 +199,11 @@ class Femnist(Dataset):
machine_id : int
Machine ID
mapping : decentralizepy.mappings.Mapping
- Mapping to conver rank, machine_id -> uid for data partitioning
- n_procs : int, optional
- The number of processes among which to divide the data. Default value is assigned 1
+ Mapping to convert rank, machine_id -> uid for data partitioning
+ It also provides the total number of global processes
train_dir : str, optional
Path to the training data files. Required to instantiate the training set
- The training set is partitioned according to n_procs and sizes
+ The training set is partitioned according to the number of global processes and sizes
test_dir : str. optional
Path to the testing data files Required to instantiate the testing set
sizes : list(int), optional
@@ -217,7 +217,6 @@ class Femnist(Dataset):
rank,
machine_id,
mapping,
- n_procs,
train_dir,
test_dir,
sizes,
diff --git a/src/decentralizepy/mappings/Mapping.py b/src/decentralizepy/mappings/Mapping.py
index b979eb9..9f764ed 100644
--- a/src/decentralizepy/mappings/Mapping.py
+++ b/src/decentralizepy/mappings/Mapping.py
@@ -18,6 +18,18 @@ class Mapping:
"""
self.n_procs = n_procs
+ def get_n_procs(self):
+ """
+ Gives the global sum of all processes that are spawned on the machines
+
+ Returns
+ -------
+ int
+ the number of global processes
+ """
+
+ return self.n_procs
+
def get_uid(self, rank: int, machine_id: int):
"""
Gives the global unique identifier of the node
diff --git a/src/decentralizepy/node/Node.py b/src/decentralizepy/node/Node.py
index e7aa916..0ef54cd 100644
--- a/src/decentralizepy/node/Node.py
+++ b/src/decentralizepy/node/Node.py
@@ -394,8 +394,6 @@ class Node:
Rank of process local to the machine
machine_id : int
Machine ID on which the process in running
- n_procs_local : int
- Number of processes on current machine
mapping : decentralizepy.mappings
The object containing the mapping rank <--> uid
graph : decentralizepy.graphs
--
GitLab