Skip to content
Snippets Groups Projects
Commit d81ca50b authored by Rishi Sharma's avatar Rishi Sharma
Browse files

6 machine, move to eval

parent 62f6aa70
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 2269 additions and 603 deletions
36_nodes.edges → eval/36_nodes.edges
+ 0
− 0
View file @ d81ca50b
File moved
eval/80_nodes.edges 0 → 100644
+ 507
− 0
View file @ d81ca50b
80
0 1
0 3
0 60
0 47
0 79
0 21
0 53
0 28
1 0
1 2
1 70
1 20
1 61
1 30
2 33
2 1
2 3
2 4
2 68
3 0
3 2
3 4
3 14
3 19
3 20
3 25
3 59
4 2
4 3
4 5
4 78
4 16
5 4
5 6
5 7
5 10
5 76
5 79
5 24
5 60
6 17
6 26
6 5
6 7
7 5
7 6
7 8
7 54
7 61
8 7
8 9
8 16
8 19
8 57
8 63
9 68
9 8
9 10
9 77
9 54
9 24
9 27
9 30
10 69
10 5
10 9
10 11
10 45
10 47
10 55
11 10
11 19
11 12
11 71
12 37
12 74
12 11
12 13
12 63
13 65
13 71
13 39
13 76
13 45
13 14
13 12
13 51
13 53
13 54
13 63
14 32
14 64
14 3
14 70
14 13
14 15
14 51
14 23
14 60
15 69
15 40
15 44
15 14
15 16
15 19
15 53
15 22
15 27
16 4
16 70
16 8
16 77
16 15
16 17
16 60
16 31
17 32
17 6
17 40
17 16
17 18
18 32
18 46
18 17
18 19
18 20
18 24
19 3
19 8
19 11
19 75
19 78
19 15
19 18
19 20
19 21
19 55
19 58
20 1
20 65
20 3
20 70
20 18
20 19
20 21
20 22
21 0
21 38
21 46
21 19
21 20
21 22
21 24
22 15
22 51
22 20
22 21
22 23
22 63
23 36
23 14
23 79
23 22
23 24
24 5
24 38
24 37
24 9
24 18
24 21
24 23
24 25
24 61
25 66
25 3
25 58
25 40
25 24
25 26
26 6
26 53
26 25
26 27
26 29
27 71
27 9
27 15
27 48
27 52
27 54
27 26
27 28
27 62
28 0
28 40
28 46
28 56
28 27
28 29
29 48
29 26
29 28
29 30
30 65
30 1
30 34
30 68
30 9
30 29
30 31
31 16
31 32
31 43
31 30
32 33
32 38
32 41
32 44
32 14
32 17
32 18
32 57
32 31
33 32
33 2
33 34
33 69
33 54
33 63
34 38
34 33
34 35
34 30
35 34
35 36
35 38
35 39
35 46
36 35
36 37
36 42
36 46
36 54
36 23
37 36
37 38
37 12
37 76
37 24
38 32
38 34
38 35
38 37
38 39
38 21
38 24
39 40
39 35
39 13
39 38
40 39
40 41
40 15
40 17
40 55
40 25
40 28
41 32
41 42
41 40
41 79
42 64
42 36
42 41
42 43
42 50
42 53
42 55
42 58
43 64
43 68
43 42
43 44
43 31
44 32
44 43
44 45
44 15
44 51
44 62
45 72
45 10
45 44
45 13
45 46
45 50
46 35
46 36
46 76
46 45
46 77
46 47
46 18
46 50
46 21
46 28
47 0
47 10
47 74
47 46
47 48
48 49
48 27
48 29
48 47
49 64
49 48
49 50
49 52
49 54
50 64
50 42
50 76
50 45
50 46
50 49
50 51
51 67
51 72
51 44
51 13
51 14
51 50
51 52
51 22
51 55
52 76
52 49
52 51
52 53
52 27
53 0
53 68
53 42
53 13
53 15
53 52
53 54
53 55
53 26
54 33
54 36
54 7
54 9
54 13
54 49
54 53
54 55
54 57
54 27
55 40
55 10
55 42
55 51
55 19
55 53
55 54
55 56
56 57
56 28
56 55
57 32
57 68
57 8
57 54
57 56
57 58
58 42
58 19
58 25
58 59
58 57
59 64
59 58
59 3
59 60
60 0
60 5
60 74
60 14
60 16
60 59
60 61
61 1
61 68
61 7
61 76
61 24
61 60
61 62
62 70
62 44
62 27
62 61
62 63
63 64
63 33
63 66
63 68
63 8
63 74
63 12
63 13
63 78
63 22
63 62
64 65
64 70
64 42
64 43
64 14
64 49
64 50
64 59
64 63
65 64
65 66
65 13
65 20
65 30
66 65
66 25
66 67
66 63
67 66
67 51
67 68
68 2
68 67
68 69
68 9
68 73
68 43
68 53
68 57
68 61
68 30
68 63
69 33
69 68
69 70
69 10
69 15
70 64
70 1
70 69
70 71
70 14
70 16
70 20
70 62
71 70
71 72
71 11
71 13
71 27
72 73
72 51
72 45
72 71
73 72
73 74
73 68
74 73
74 75
74 12
74 47
74 60
74 63
75 74
75 19
75 76
76 37
76 5
76 75
76 13
76 46
76 77
76 50
76 52
76 61
77 9
77 76
77 46
77 78
77 16
78 4
78 77
78 79
78 19
78 63
79 0
79 5
79 41
79 78
79 23
eval/96_nodes.edges 0 → 100644
+ 531
− 0
View file @ d81ca50b
96
0 1
0 66
0 8
0 43
0 19
0 58
0 95
1 0
1 2
1 35
1 4
1 80
1 50
1 90
2 56
2 3
2 35
2 1
3 2
3 4
3 5
3 72
3 15
3 86
3 55
4 1
4 3
4 36
4 37
4 38
4 5
4 76
5 3
5 4
5 6
5 49
5 53
5 92
6 67
6 36
6 5
6 7
6 78
6 86
7 64
7 6
7 8
7 41
7 47
7 17
7 87
8 0
8 7
8 9
8 56
8 26
9 8
9 10
9 75
9 77
9 15
10 32
10 36
10 9
10 11
10 12
10 81
10 82
11 32
11 34
11 10
11 12
11 59
11 92
11 61
12 13
12 10
12 11
12 29
13 18
13 12
13 14
14 73
14 91
14 13
14 15
15 3
15 9
15 75
15 14
15 47
15 16
15 27
15 31
16 17
16 66
16 46
16 15
17 16
17 18
17 20
17 7
18 32
18 13
18 79
18 17
18 19
18 93
19 0
19 18
19 20
19 86
20 46
20 80
20 17
20 19
20 21
20 88
20 90
21 20
21 69
21 22
21 23
22 35
22 69
22 79
22 21
22 23
22 58
23 38
23 77
23 21
23 22
23 24
23 89
23 58
24 25
24 58
24 23
24 79
25 36
25 69
25 41
25 42
25 24
25 26
26 8
26 25
26 27
26 87
27 34
27 26
27 28
27 15
28 27
28 46
28 82
28 91
28 29
28 95
29 12
29 28
29 53
29 56
29 60
29 30
30 35
30 45
30 92
30 29
30 31
31 64
31 33
31 32
31 76
31 78
31 15
31 50
31 30
32 33
32 37
32 10
32 11
32 42
32 18
32 31
33 32
33 34
33 31
34 33
34 35
34 75
34 11
34 55
34 27
35 1
35 2
35 34
35 36
35 53
35 22
35 56
35 30
36 89
36 35
36 4
36 37
36 6
36 72
36 10
36 75
36 85
36 25
37 32
37 4
37 36
37 38
37 51
38 4
38 37
38 39
38 43
38 23
39 38
39 40
39 42
39 75
39 94
40 70
40 39
40 41
40 48
40 49
40 54
40 95
41 68
41 7
41 40
41 42
41 43
41 25
41 91
42 32
42 70
42 39
42 41
42 43
42 56
42 25
42 60
43 0
43 38
43 41
43 42
43 44
44 64
44 71
44 43
44 45
44 58
45 46
45 44
45 30
46 45
46 47
46 16
46 48
46 20
46 28
47 48
47 15
47 46
47 7
48 40
48 46
48 47
48 49
48 89
48 62
48 63
49 5
49 71
49 40
49 78
49 48
49 50
50 1
50 51
50 49
50 31
51 67
51 37
51 73
51 50
51 52
52 53
52 51
52 59
52 69
53 35
53 5
53 52
53 54
53 55
53 89
53 29
53 94
54 40
54 82
54 84
54 53
54 55
55 34
55 3
55 53
55 54
55 56
56 2
56 35
56 8
56 42
56 55
56 57
56 29
57 75
57 56
57 58
57 91
58 0
58 44
58 22
58 23
58 24
58 57
58 59
59 58
59 11
59 52
59 60
60 59
60 42
60 61
60 29
61 70
61 11
61 60
61 62
62 79
62 48
62 81
62 86
62 90
62 61
62 63
63 48
63 88
63 64
63 62
64 65
64 7
64 44
64 63
64 31
65 72
65 66
65 64
66 0
66 16
66 67
66 65
67 66
67 68
67 6
67 76
67 51
68 41
68 67
68 69
68 86
69 68
69 70
69 52
69 21
69 22
69 25
69 92
70 69
70 71
70 40
70 42
70 75
70 61
71 70
71 72
71 44
71 49
71 89
71 94
72 65
72 3
72 36
72 71
72 73
73 72
73 74
73 51
73 14
74 73
74 75
74 86
75 34
75 36
75 70
75 39
75 9
75 74
75 76
75 15
75 57
76 67
76 4
76 75
76 77
76 92
76 31
77 9
77 76
77 78
77 23
78 6
78 77
78 79
78 49
78 31
79 78
79 80
79 18
79 22
79 24
79 62
80 1
80 79
80 81
80 82
80 20
80 87
81 80
81 10
81 82
81 62
82 10
82 80
82 81
82 83
82 84
82 54
82 28
82 93
83 82
83 84
83 93
84 82
84 83
84 85
84 54
84 88
84 90
85 36
85 86
85 84
86 3
86 68
86 6
86 74
86 19
86 85
86 87
86 93
86 62
87 7
87 80
87 86
87 88
87 26
88 20
88 84
88 87
88 89
88 63
89 36
89 71
89 48
89 53
89 23
89 88
89 90
90 1
90 20
90 84
90 89
90 91
90 62
91 41
91 14
91 92
91 57
91 90
91 28
92 69
92 5
92 11
92 76
92 91
92 93
92 30
93 18
93 83
93 82
93 86
93 92
93 94
94 71
94 39
94 53
94 93
94 95
95 40
95 0
95 28
95 94
config.ini → eval/config.ini
+ 4
− 4
View file @ d81ca50b
......@@ -2,9 +2,9 @@
dataset_package = decentralizepy.datasets.Femnist
dataset_class = Femnist
model_class = CNN
n_procs = 12
train_dir = leaf/data/femnist/per_user_data/train
test_dir = leaf/data/femnist/data/test
n_procs = 16
train_dir = /home/risharma/Gitlab/decentralizepy/leaf/data/femnist/per_user_data/train
test_dir = /home/risharma/Gitlab/decentralizepy/leaf/data/femnist/data/test
; python list of fractions below
sizes =
......@@ -25,7 +25,7 @@ loss_class = CrossEntropyLoss
[COMMUNICATION]
comm_package = decentralizepy.communication.TCP
comm_class = TCP
addresses_filepath = ip_addr.json
addresses_filepath = ip_addr_6Machines.json
[SHARING]
sharing_package = decentralizepy.sharing.Sharing
......
graph.adj → eval/graph.adj
+ 0
− 0
View file @ d81ca50b
File moved
ip_addr.json → eval/ip_addr_3Machines.json
+ 0
− 0
View file @ d81ca50b
File moved
eval/ip_addr_6Machines.json 0 → 100644
+ 8
− 0
View file @ d81ca50b
{
"0": "10.90.41.128",
"1": "10.90.41.129",
"2": "10.90.41.130",
"3": "10.90.41.131",
"4": "10.90.41.132",
"5": "10.90.41.133"
}
\ No newline at end of file
eval/main.ipynb 0 → 100644
+ 1199
− 0
View file @ d81ca50b
This diff is collapsed.
testing.py → eval/testing.py
+ 0
− 0
View file @ d81ca50b
File moved
install_nMachines.sh 0 → 100644
+ 14
− 0
View file @ d81ca50b
#!\bin\bash
cd
mkdir -p Gitlab
cd Gitlab
git clone git@gitlab.epfl.ch:risharma/decentralizepy.git
cd decentralizepy
mkdir -p leaf/data/femnist/data/train
mkdir -p leaf/data/femnist/data/test
mkdir -p leaf/data/femnist/per_user_data/train
~/miniconda3/bin/conda remove --name decpy --all
~/miniconda3/bin/conda create -n decpy python=3.9
~/miniconda3/envs/decpy/bin/pip install --upgrade pip --quiet
~/miniconda3/envs/decpy/bin/pip install --editable .\[dev\]
main.ipynb deleted 100644 → 0
+ 0
− 585
View file @ 62f6aa70
%% Cell type:code id: tags:
```
from datasets.Femnist import Femnist
from graphs import SmallWorld
from collections import defaultdict
import os
import json
import numpy as np
```
%% Cell type:code id: tags:
```
a = FEMNIST
a
```
%% Cell type:code id: tags:
```
b = SmallWorld(6, 2, 2, 1)
```
%% Cell type:code id: tags:
```
b.adj_list
```
%% Cell type:code id: tags:
```
for i in range(12):
print(b.neighbors(i))
```
%% Cell type:code id: tags:
```
clients = []
```
%% Cell type:code id: tags:
```
num_samples = []
data = defaultdict(lambda : None)
```
%% Cell type:code id: tags:
```
datadir = "./leaf/data/femnist/data/train"
files = os.listdir(datadir)
total_users=0
users = set()
```
%% Cell type:code id: tags:
```
files = os.listdir(datadir)[0:1]
```
%% Cell type:code id: tags:
```
for f in files:
file_path = os.path.join(datadir, f)
print(file_path)
with open(file_path, 'r') as inf:
client_data = json.load(inf)
current_users = len(client_data['users'])
print("Current_Users: ", current_users)
total_users += current_users
users.update(client_data['users'])
print("total_users: ", total_users)
print("total_users: ", len(users))
print(client_data['user_data'].keys())
print(np.array(client_data['user_data']['f3408_47']['x']).shape)
print(np.array(client_data['user_data']['f3408_47']['y']).shape)
print(np.array(client_data['user_data']['f3327_11']['x']).shape)
print(np.array(client_data['user_data']['f3327_11']['y']).shape)
print(np.unique(np.array(client_data['user_data']['f3327_11']['y'])))
```
%% Cell type:code id: tags:
```
file = 'run.py'
with open(file, 'r') as inf:
print(inf.readline().strip())
print(inf.readlines())
```
%% Cell type:code id: tags:
```
def f(l):
l[2] = 'c'
a = ['a', 'a', 'a']
print(a)
f(a)
print(a)
```
%% Cell type:code id: tags:
```
l = ['a', 'b', 'c']
print(l[:-1])
```
%% Cell type:code id: tags:
```
from localconfig import LocalConfig
def read_ini(file_path):
config = LocalConfig(file_path)
for section in config:
print("Section: ", section)
for key, value in config.items(section):
print((key, value))
print(dict(config.items('DATASET')))
return config
config = read_ini("config.ini")
for section in config:
print(section)
#d = dict(config.sections())
```
%% Cell type:code id: tags:
```
def func(a = 1, b = 2, c = 3):
print(a + b + c)
l = [3, 5, 7]
func(*l)
```
%% Cell type:code id: tags:
```
from torch import multiprocessing as mp
mp.spawn(fn = func, nprocs = 2, args = [], kwargs = {'a': 4, 'b': 5, 'c': 6})
```
%% Cell type:code id: tags:
```
l = '[0.4, 0.2, 0.3, 0.1]'
type(eval(l))
```
%% Cell type:code id: tags:
```
from decentralizepy.datasets.Femnist import Femnist
f1 = Femnist(0, 1, 'leaf/data/femnist/data/train')
ts = f1.get_trainset(1)
for data, target in ts:
print(data)
break
```
%% Cell type:code id: tags:
```
from decentralizepy.datasets.Femnist import Femnist
from decentralizepy.graphs.SmallWorld import SmallWorld
from decentralizepy.mappings.Linear import Linear
f = Femnist(2, 'leaf/data/femnist/data/train', sizes=[0.6, 0.4])
g = SmallWorld(4, 1, 0.5)
l = Linear(2, 2)
```
%% Cell type:code id: tags:
```
from decentralizepy.node.Node import Node
from torch import multiprocessing as mp
import logging
n1 = Node(0, l, g, f, "./results", logging.DEBUG)
n2 = Node(1, l, g, f, "./results", logging.DEBUG)
# mp.spawn(fn = Node, nprocs = 2, args=[l,g,f])
```
%% Cell type:code id: tags:
```
from testing import f
```
%% Cell type:code id: tags:
```
from torch import multiprocessing as mp
import torch
m1 = torch.nn.Linear(1,1)
o1 = torch.optim.SGD(m1.parameters(), 0.6)
print(m1)
mp.spawn(fn = f, nprocs = 2, args=[m1, o1])
```
%% Cell type:markdown id: tags:
%% Cell type:code id: tags:
```
o1.param_groups
```
%% Cell type:code id: tags:
```
with torch.no_grad():
o1.param_groups[0]["params"][0].copy_(torch.zeros(1,))
```
%% Cell type:code id: tags:
```
o1.param_groups
```
%% Cell type:code id: tags:
```
m1.state_dict()
```
%% Cell type:code id: tags:
```
import torch
loss = getattr(torch.nn.functional, 'nll_loss')
```
%% Cell type:code id: tags:
```
loss
```
%% Cell type:code id: tags:
```
%matplotlib inline
from decentralizepy.node.Node import Node
from decentralizepy.graphs.SmallWorld import SmallWorld
from decentralizepy.graphs.Graph import Graph
from decentralizepy.mappings.Linear import Linear
from torch import multiprocessing as mp
import torch
import logging
from localconfig import LocalConfig
def read_ini(file_path):
config = LocalConfig(file_path)
for section in config:
print("Section: ", section)
for key, value in config.items(section):
print((key, value))
print(dict(config.items('DATASET')))
return config
config = read_ini("config.ini")
my_config = dict()
for section in config:
my_config[section] = dict(config.items(section))
#f = Femnist(2, 'leaf/data/femnist/data/train', sizes=[0.6, 0.4])
g = Graph()
g.read_graph_from_file("36_nodes.edges", "edges")
l = Linear(1, 36)
#Node(0, 0, l, g, my_config, 20, "results", logging.DEBUG)
mp.spawn(fn = Node, nprocs = g.n_procs, args=[0,l,g,my_config,20,"results",logging.INFO])
# mp.spawn(fn = Node, args = [l, g, config, 10, "results", logging.DEBUG], nprocs=2)
```
%% Output
Section: GRAPH
('package', 'decentralizepy.graphs.SmallWorld')
('graph_class', 'SmallWorld')
Section: DATASET
('dataset_package', 'decentralizepy.datasets.Femnist')
('dataset_class', 'Femnist')
('model_class', 'CNN')
('n_procs', 36)
('train_dir', 'leaf/data/femnist/per_user_data/train')
('test_dir', 'leaf/data/femnist/data/test')
('sizes', '')
Section: OPTIMIZER_PARAMS
('optimizer_package', 'torch.optim')
('optimizer_class', 'Adam')
('lr', 0.01)
Section: TRAIN_PARAMS
('training_package', 'decentralizepy.training.Training')
('training_class', 'Training')
('epochs_per_round', 1)
('batch_size', 1024)
('shuffle', True)
('loss_package', 'torch.nn')
('loss_class', 'CrossEntropyLoss')
Section: COMMUNICATION
('comm_package', 'decentralizepy.communication.TCP')
('comm_class', 'TCP')
('addresses_filepath', 'ip_addr.json')
Section: SHARING
('sharing_package', 'decentralizepy.sharing.Sharing')
('sharing_class', 'Sharing')
{'dataset_package': 'decentralizepy.datasets.Femnist', 'dataset_class': 'Femnist', 'model_class': 'CNN', 'n_procs': 36, 'train_dir': 'leaf/data/femnist/per_user_data/train', 'test_dir': 'leaf/data/femnist/data/test', 'sizes': ''}
%% Cell type:code id: tags:
```
```
%% Cell type:code id: tags:
```
from decentralizepy.mappings.Linear import Linear
from testing import f
from torch import multiprocessing as mp
l = Linear(1, 2)
mp.spawn(fn = f, nprocs = 2, args = [0, 2, "ip_addr.json", l])
```
%% Cell type:code id: tags:
```
from decentralizepy.datasets.Femnist import Femnist
f = Femnist()
f.file_per_user('leaf/data/femnist/data/train','leaf/data/femnist/per_user_data/train')
```
%% Cell type:code id: tags:
```
a = set()
a.update([2, 3, 4, 5])
```
%% Cell type:code id: tags:
```
a
```
%% Output
{2, 3, 4, 5}
%% Cell type:code id: tags:
```
print(*a)
```
%% Output
2 3 4 5
%% Cell type:code id: tags:
```
from decentralizepy.graphs.SmallWorld import SmallWorld
s = SmallWorld(36, 2, .5)
s.write_graph_to_file('36_nodes.edges')
```
%% Cell type:code id: tags:
```
import sys
sys.argv
```
%% Output
['/home/risharma/miniconda3/envs/decpy/lib/python3.9/site-packages/ipykernel_launcher.py',
'--ip=127.0.0.1',
'--stdin=9008',
'--control=9006',
'--hb=9005',
'--Session.signature_scheme="hmac-sha256"',
'--Session.key=b"eac5d2f8-c460-45f1-a268-1e4b46a6efd6"',
'--shell=9007',
'--transport="tcp"',
'--iopub=9009',
'--f=/tmp/tmp-21212479paJaUBJBN84.json']
%% Cell type:code id: tags:
```
```
run.py deleted 100644 → 0
+ 0
− 14
View file @ 62f6aa70
import torch
import torch.multiprocessing as mp
x = [1, 2]
def f(id, a):
print(id, x)
print(id, a)
if __name__ == "__main__":
x.append(3)
mp.spawn(f, nprocs=2, args=(x,))
src/decentralizepy/sharing/PartialModel.py
+ 6
− 0
View file @ d81ca50b
from decentralizepy.sharing.Sharing import Sharing
class PartialModel(Sharing):
def __init__(self, rank, machine_id, communication, mapping, graph, model, dataset):
super().__init__(rank, machine_id, communication, mapping, graph, model, dataset)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment