diff --git a/eval/.ipynb_checkpoints/ip_addr_7Machines-checkpoint.json b/eval/.ipynb_checkpoints/ip_addr_7Machines-checkpoint.json
new file mode 100644
index 0000000000000000000000000000000000000000..889afa03d3f0173318e51c13c3f3f2a17cc7c88e
--- /dev/null
+++ b/eval/.ipynb_checkpoints/ip_addr_7Machines-checkpoint.json
@@ -0,0 +1,9 @@
+{
+ "0": "10.90.41.127",
+ "1": "10.90.41.128",
+ "2": "10.90.41.129",
+ "3": "10.90.41.130",
+ "4": "10.90.41.131",
+ "5": "10.90.41.132",
+ "6": "10.90.41.133"
+}
\ No newline at end of file
diff --git a/eval/16_regular.edges b/eval/16_regular.edges
new file mode 100644
index 0000000000000000000000000000000000000000..4e847b9534add5352272851013ac3c1994f09b43
--- /dev/null
+++ b/eval/16_regular.edges
@@ -0,0 +1,49 @@
+16
+0 1
+0 3
+0 15
+1 0
+1 10
+1 2
+2 1
+2 3
+2 15
+3 0
+3 2
+3 4
+4 9
+4 3
+4 5
+5 4
+5 13
+5 6
+6 11
+6 5
+6 7
+7 8
+7 14
+7 6
+8 9
+8 12
+8 7
+9 8
+9 10
+9 4
+10 1
+10 11
+10 9
+11 10
+11 12
+11 6
+12 8
+12 11
+12 13
+13 12
+13 5
+13 14
+14 15
+14 13
+14 7
+15 0
+15 2
+15 14
diff --git a/eval/ip_addr_1Machines.json b/eval/ip_addr_1Machines.json
new file mode 100644
index 0000000000000000000000000000000000000000..15d6591df53574707ac03627fa19c9ecd749b1e3
--- /dev/null
+++ b/eval/ip_addr_1Machines.json
@@ -0,0 +1,3 @@
+{
+ "0": "127.0.0.1"
+}
\ No newline at end of file
diff --git a/eval/run_grid1.sh b/eval/run_grid1.sh
new file mode 100755
index 0000000000000000000000000000000000000000..e8e6659b39927f75896a98347d8a2d0fea8f69a0
--- /dev/null
+++ b/eval/run_grid1.sh
@@ -0,0 +1,107 @@
+#!/bin/bash
+# Documentation
+# This bash file takes three inputs. The first argument (nfs_home) is the path to the nfs home directory.
+# The second one (python_bin) is the path to the python bin folder.
+# The last argument (logs_subfolder) is the path to the logs folder with respect to the nfs home directory.
+#
+# The nfs home directory should contain the code of this framework stored in $nfs_home/decentralizepy and a folder
+# called configs which contains the file 'ip_addr_6Machines.json'
+# The python bin folder needs to include all the dependencies of this project including crudini.
+# The results will be stored in $nfs_home/$logs_subfolder
+# Each of the experiments will be stored in its own folder inside the logs_subfolder. The folder of the experiment
+# starts with the last part of the config name, i.e., for 'config_celeba_topkacc.ini' it will start with topkacc.
+# The name further includes the learning rate, rounds and batchsize as well as the exact date at which the experiment
+# was run.
+# Example: ./run_grid.sh /mnt/nfs/wigger /mnt/nfs/wigger/anaconda3/envs/sacs39/bin /logs/celaba
+#
+# Additional requirements:
+# Each node needs a folder called 'tmp' in the user's home directory
+#
+# Note:
+# - The script does not change the optimizer. All configs are writen to use Adam.
+# For SGD these need to be changed manually
+# - The script will set '--test_after' and '--train_evaluate_after' to comm_rounds_per_global_epoch, i.e., the eavaluation
+# on the train set and on the test set is carried out every global epoch.
+# - The '--reset_optimizer' option is set to 0, i.e., the optimizer is not reset after a communication round (only
+# relevant for Adams and other optimizers with internal state)
+#
+# Addapting the script to other datasets:
+# Change the variable 'dataset_size' to reflect the data sets size.
+#
+# Known issues:
+# - If the script is started at the very end of a minute then there is a change that two folders are created as not all
+# machines may start running the script at the exact same moment.
+
+nfs_home=$1
+python_bin=$2
+logs_subfolder=$3
+decpy_path=$nfs_home/decentralizepy/eval
+cd $decpy_path
+
+env_python=$python_bin/python3
+graph=192_regular.edges
+config_file=~/tmp/config.ini
+procs_per_machine=32
+machines=6
+global_epochs=25
+eval_file=testing.py
+log_level=INFO
+
+ip_machines=$nfs_home/configs/ip_addr_6Machines.json
+
+m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print $2}') | cut -d'"' -f2`
+export PYTHONFAULTHANDLER=1
+
+# Base configs for which the gird search is done
+tests=("step_configs/config_celeba_sharing.ini")
+# Learning rates to test
+lrs=( "0.1" "0.01" "0.001")
+
+batchsize=("8" "16")
+comm_rounds_per_global_epoch=("1" "10" "100")
+# Celeba has 63741 samples
+procs=`expr $procs_per_machine \* $machines`
+echo procs: $procs
+dataset_size=63741
+samples_per_user=`expr $dataset_size / $procs`
+echo samples per user: $samples_per_user
+
+for b in "${batchsize[@]}"
+do
+ echo batchsize: $b
+ for r in "${comm_rounds_per_global_epoch[@]}"
+ do
+ echo communication rounds per global epoch: $r
+ batches_per_epoch=$(($samples_per_user / $b))
+ echo batches per global epoch: $batches_per_epoch
+ iterations=$(($global_epochs * $r))
+ echo iterations: $iterations
+ batches_per_comm_round=$($env_python -c "from math import floor; x = floor($batches_per_epoch / $r); print(1 if x==0 else x)")
+ new_iterations=$($env_python -c "from math import floor; x = floor($batches_per_epoch / $r); y = floor((($batches_per_epoch / $r) -x +1)*$iterations); print($iterations if x==0 else y)")
+ echo batches per communication round: $batches_per_comm_round
+ echo corrected iterations: $new_iterations
+ for lr in "${lrs[@]}"
+ do
+ for i in "${tests[@]}"
+ do
+ echo $i
+ IFS='_' read -ra NAMES <<< $i
+ IFS='.' read -ra NAME <<< ${NAMES[-1]}
+ log_dir=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$r:b=$b:$(date '+%Y-%m-%dT%H:%M')/machine$m
+ echo results are stored in: $log_dir
+ mkdir -p $log_dir
+ cp $i $config_file
+ $python_bin/crudini --set $config_file COMMUNICATION addresses_filepath $ip_machines
+ $python_bin/crudini --set $config_file OPTIMIZER_PARAMS lr $lr
+ $python_bin/crudini --set $config_file TRAIN_PARAMS rounds $batches_per_comm_round
+ $python_bin/crudini --set $config_file TRAIN_PARAMS batch_size $b
+ $env_python $eval_file -ro 0 -tea $r -ld $log_dir -mid $m -ps $procs_per_machine -ms $machines -is $new_iterations -gf $graph -ta $r -cf $config_file -ll $log_level
+ echo $i is done
+ sleep 1
+ echo end of sleep
+ done
+ done
+ done
+done
+#
+
diff --git a/eval/run_grid_server.sh b/eval/run_grid_server.sh
new file mode 100755
index 0000000000000000000000000000000000000000..b5b9982e15509446fc65c04f5fc7bb9f7b45e8b0
--- /dev/null
+++ b/eval/run_grid_server.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+# Documentation
+# This bash file takes three inputs. The first argument (nfs_home) is the path to the nfs home directory.
+# The second one (python_bin) is the path to the python bin folder.
+# The last argument (logs_subfolder) is the path to the logs folder with respect to the nfs home directory.
+#
+# The nfs home directory should contain the code of this framework stored in $nfs_home/decentralizepy and a folder
+# called configs which contains the file 'ip_addr_6Machines.json'
+# The python bin folder needs to include all the dependencies of this project including crudini.
+# The results will be stored in $nfs_home/$logs_subfolder
+# Each of the experiments will be stored in its own folder inside the logs_subfolder. The folder of the experiment
+# starts with the last part of the config name, i.e., for 'config_celeba_topkacc.ini' it will start with topkacc.
+# The name further includes the learning rate, rounds and batchsize as well as the exact date at which the experiment
+# was run.
+# Example:
+#
+# Note:
+# - The script does not change the optimizer. All configs are writen to use Adam.
+# For SGD these need to be changed manually
+# - The script will set '--test_after' and '--train_evaluate_after' to comm_rounds_per_global_epoch, i.e., the eavaluation
+# on the train set and on the test set is carried out every global epoch.
+# - The '--reset_optimizer' option is set to 0, i.e., the optimizer is not reset after a communication round (only
+# relevant for Adams and other optimizers with internal state)
+#
+#
+# Known issues:
+# - If the script is started at the very end of a minute then there is a change that two folders are created as not all
+# machines may start running the script at the exact same moment.
+
+nfs_home=$1
+python_bin=$2
+logs_subfolder=$3
+decpy_path=$nfs_home/decentralizepy/eval
+cd $decpy_path
+
+env_python=$python_bin/python3
+graph=192_regular.edges #4_node_fullyConnected.edges
+config_file=~/tmp/config.ini
+procs_per_machine=32
+machines=6
+global_epochs=1
+eval_file=testing.py
+log_level=INFO
+
+ip_machines=$nfs_home/configs/ip_addr_6Machines.json
+
+m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print $2}') | cut -d'"' -f2`
+export PYTHONFAULTHANDLER=1
+
+# Base configs for which the gird search is done
+tests=("step_configs/config_celeba_sharing.ini")
+# Learning rates to test
+lrs=( "0.1" "0.01")
+
+batchsize=("8" "16")
+comm_rounds_per_global_epoch=("1" "10")
+# Celeba has 63741 samples
+procs=`expr $procs_per_machine \* $machines`
+echo procs: $procs
+dataset_size=63741
+samples_per_user=`expr $dataset_size / $procs`
+echo samples per user: $samples_per_user
+
+for b in "${batchsize[@]}"
+do
+ echo batchsize: $b
+ for r in "${comm_rounds_per_global_epoch[@]}"
+ do
+ echo communication rounds per global epoch: $r
+ batches_per_epoch=$(($samples_per_user / $b))
+ echo batches per global epoch: $batches_per_epoch
+ iterations=$(($global_epochs * $r))
+ echo iterations: $iterations
+ batches_per_comm_round=$($env_python -c "from math import floor; x = floor($batches_per_epoch / $r); print(1 if x==0 else x)")
+ new_iterations=$($env_python -c "from math import floor; x = floor($batches_per_epoch / $r); y = floor((($batches_per_epoch / $r) -x +1)*$iterations); print($iterations if x==0 else y)")
+ echo batches per communication round: $batches_per_comm_round
+ echo corrected iterations: $new_iterations
+ for lr in "${lrs[@]}"
+ do
+ for i in "${tests[@]}"
+ do
+ echo $i
+ IFS='_' read -ra NAMES <<< $i
+ IFS='.' read -ra NAME <<< ${NAMES[-1]}
+ log_dir=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$r:b=$b:$(date '+%Y-%m-%dT%H:%M')/machine$m
+ echo results are stored in: $log_dir
+ mkdir -p $log_dir
+ cp $i $config_file
+ $python_bin/crudini --set $config_file COMMUNICATION addresses_filepath $ip_machines
+ $python_bin/crudini --set $config_file OPTIMIZER_PARAMS lr $lr
+ $python_bin/crudini --set $config_file TRAIN_PARAMS rounds $batches_per_comm_round
+ $python_bin/crudini --set $config_file TRAIN_PARAMS batch_size $b
+ $env_python $eval_file -ro 0 -tea $r -ld $log_dir -mid $m -ps $procs_per_machine -ms $machines -is $new_iterations -gf $graph -ta $r -cf $config_file -ll $log_level
+ echo $i is done
+ sleep 1
+ echo end of sleep
+ done
+ done
+ done
+done
+#
+
diff --git a/eval/run_reddit_local2.sh b/eval/run_reddit_local2.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5bcebde3c2729c1491f257fcdf03cacf81088abe
--- /dev/null
+++ b/eval/run_reddit_local2.sh
@@ -0,0 +1,115 @@
+#!/bin/bash
+# Documentation
+# This bash file takes three inputs. The first argument (nfs_home) is the path to the nfs home directory.
+# The second one (python_bin) is the path to the python bin folder.
+# The last argument (logs_subfolder) is the path to the logs folder with respect to the nfs home directory.
+#
+# The nfs home directory should contain the code of this framework stored in $nfs_home/decentralizepy and a folder
+# called configs which contains the file 'ip_addr_6Machines.json'
+# The python bin folder needs to include all the dependencies of this project including crudini.
+# The results will be stored in $nfs_home/$logs_subfolder
+# Each of the experiments will be stored in its own folder inside the logs_subfolder. The folder of the experiment
+# starts with the last part of the config name, i.e., for 'config_celeba_topkacc.ini' it will start with topkacc.
+# The name further includes the learning rate, rounds and batchsize as well as the exact date at which the experiment
+# was run.
+# Example: ./run_grid.sh /mnt/nfs/wigger /mnt/nfs/wigger/anaconda3/envs/sacs39/bin /logs/celeba
+#
+# Additional requirements:
+# Each node needs a folder called 'tmp' in the user's home directory
+#
+# Note:
+# - The script does not change the optimizer. All configs are writen to use SGD.
+# - The script will set '--test_after' and '--train_evaluate_after' such that it happens at the end of a global epoch.
+# - The '--reset_optimizer' option is set to 0, i.e., the optimizer is not reset after a communication round (only
+# relevant for Adams and other optimizers with internal state)
+#
+# Addapting the script to other datasets:
+# Change the variable 'dataset_size' to reflect the data sets size.
+#
+# Known issues:
+# - If the script is started at the very end of a minute then there is a change that two folders are created as not all
+# machines may start running the script at the exact same moment.
+
+nfs_home=/tmp/logs/
+python_bin=/home/jeffrey/anaconda3/envs/sacs39/bin
+logs_subfolder=reddit_local
+
+env_python=$python_bin/python3
+graph=16_regular.edges
+config_file=~/tmp/config.ini
+procs_per_machine=16
+machines=1
+global_epochs=2
+eval_file=testing.py
+log_level=DEBUG
+
+ip_machines=ip_addr_1Machines.json
+
+m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print $2}') | cut -d'"' -f2`
+
+# Base configs for which the gird search is done
+tests=("step_configs/config_reddit_jwins+_local.ini") #("step_configs/config_reddit_sharing_local.ini")
+# Learning ratesJwinsDynamicGraph.py
+lr="1"
+# Batch size
+batchsize="16"
+# The number of communication rounds per global epoch
+comm_rounds_per_global_epoch="10"
+procs=`expr $procs_per_machine \* $machines`
+echo procs: $procs
+# Celeba has 63741 samples
+# Reddit has 70642
+# Femnist 734463
+# Shakespeares 3678451
+dataset_size=70642
+# Calculating the number of samples that each user/proc will have on average
+samples_per_user=`expr $dataset_size / $procs`
+echo samples per user: $samples_per_user
+
+# random_seeds for which to rerun the experiments
+random_seeds=("97")
+# random_seed = 97
+echo batchsize: $batchsize
+echo communication rounds per global epoch: $comm_rounds_per_global_epoch
+# calculating how many batches there are in a global epoch for each user/proc
+batches_per_epoch=$(($samples_per_user / $batchsize))
+echo batches per global epoch: $batches_per_epoch
+# the number of iterations in 25 global epochs
+iterations=$($env_python -c "from math import floor; print($batches_per_epoch * $global_epochs) if $comm_rounds_per_global_epoch >= $batches_per_epoch else print($global_epochs * $comm_rounds_per_global_epoch)")
+echo iterations: $iterations
+# calculating the number of batches each user/proc uses per communication step (The actual number may be a float, which we round down)
+batches_per_comm_round=$($env_python -c "from math import floor; x = floor($batches_per_epoch / $comm_rounds_per_global_epoch); print(1 if x==0 else x)")
+# since the batches per communication round were rounded down we need to change the number of iterations to reflect that
+new_iterations=$($env_python -c "from math import floor; tmp = floor($batches_per_epoch / $comm_rounds_per_global_epoch); x = 1 if tmp == 0 else tmp; y = floor((($batches_per_epoch / $comm_rounds_per_global_epoch)/x)*$iterations); print($iterations if y<$iterations else y)")
+echo batches per communication round: $batches_per_comm_round
+echo corrected iterations: $new_iterations
+test_after=10 #$(($new_iterations / $global_epochs))
+echo test after: $test_after
+for i in "${tests[@]}"
+do
+ for seed in "${random_seeds[@]}"
+ do
+ echo $i
+ IFS='_' read -ra NAMES <<< $i
+ IFS='.' read -ra NAME <<< ${NAMES[-1]}
+ log_dir_base=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')
+ echo results are stored in: $log_dir_base
+ log_dir=$log_dir_base/machine0
+ mkdir -p $log_dir
+ weight_store_dir=$log_dir_base/weights
+ mkdir -p $weight_store_dir
+ cp $i $config_file
+ # changing the config files to reflect the values of the current grid search state
+ $python_bin/crudini --set $config_file COMMUNICATION addresses_filepath $ip_machines
+ $python_bin/crudini --set $config_file OPTIMIZER_PARAMS lr $lr
+ $python_bin/crudini --set $config_file TRAIN_PARAMS rounds $batches_per_comm_round
+ $python_bin/crudini --set $config_file TRAIN_PARAMS batch_size $batchsize
+ $python_bin/crudini --set $config_file DATASET random_seed $seed
+
+ $env_python -q -X faulthandler $eval_file -ro 0 -tea $test_after -ld $log_dir -wsd $weight_store_dir -mid 0 -ps $procs_per_machine -ms $machines -is $new_iterations -gf $graph -ta $test_after -cf $config_file -ll $log_level
+ echo $i is done
+ sleep 10
+ echo end of sleep
+ done
+done
+#
\ No newline at end of file
diff --git a/eval/run_xtimes_reddit_local2.sh b/eval/run_xtimes_reddit_local2.sh
new file mode 100755
index 0000000000000000000000000000000000000000..9751bdc4be68521d3a00437cc0cf5798a33f6db9
--- /dev/null
+++ b/eval/run_xtimes_reddit_local2.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+# Documentation
+# This bash file takes three inputs. The first argument (nfs_home) is the path to the nfs home directory.
+# The second one (python_bin) is the path to the python bin folder.
+# The last argument (logs_subfolder) is the path to the logs folder with respect to the nfs home directory.
+#
+# The nfs home directory should contain the code of this framework stored in $nfs_home/decentralizepy and a folder
+# called configs which contains the file 'ip_addr_6Machines.json'
+# The python bin folder needs to include all the dependencies of this project including crudini.
+# The results will be stored in $nfs_home/$logs_subfolder
+# Each of the experiments will be stored in its own folder inside the logs_subfolder. The folder of the experiment
+# starts with the last part of the config name, i.e., for 'config_celeba_topkacc.ini' it will start with topkacc.
+# The name further includes the learning rate, rounds and batchsize as well as the exact date at which the experiment
+# was run.
+# Example: ./run_grid.sh /mnt/nfs/wigger /mnt/nfs/wigger/anaconda3/envs/sacs39/bin /logs/celeba
+#
+# Additional requirements:
+# Each node needs a folder called 'tmp' in the user's home directory
+#
+# Note:
+# - The script does not change the optimizer. All configs are writen to use SGD.
+# - The script will set '--test_after' and '--train_evaluate_after' such that it happens at the end of a global epoch.
+# - The '--reset_optimizer' option is set to 0, i.e., the optimizer is not reset after a communication round (only
+# relevant for Adams and other optimizers with internal state)
+#
+# Addapting the script to other datasets:
+# Change the variable 'dataset_size' to reflect the data sets size.
+#
+# Known issues:
+# - If the script is started at the very end of a minute then there is a change that two folders are created as not all
+# machines may start running the script at the exact same moment.
+
+nfs_home=/tmp/logs/
+python_bin=/home/jeffrey/anaconda3/envs/sacs39/bin
+logs_subfolder=reddit_local/
+
+env_python=$python_bin/python3
+graph=16_regular.edges
+config_file=~/tmp/config.ini
+procs_per_machine=16
+machines=1
+global_epochs=2
+eval_file=testing.py
+log_level=INFO
+
+ip_machines=ip_addr_1Machines.json
+
+m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print $2}') | cut -d'"' -f2`
+
+# Base configs for which the gird search is done
+tests=("step_configs/config_reddit_topkacc_local.ini") # config_reddit_sharing_local.ini") # config_reddit_subsampling_local.ini") #
+# Learning rates
+lr="1"
+# Batch size
+batchsize="16"
+# The number of communication rounds per global epoch
+comm_rounds_per_global_epoch="10"
+procs=`expr $procs_per_machine \* $machines`
+echo procs: $procs
+# Celeba has 63741 samples
+# Reddit has 70642
+# Femnist 734463
+# Shakespeares 3678451
+dataset_size=70642
+# Calculating the number of samples that each user/proc will have on average
+samples_per_user=`expr $dataset_size / $procs`
+echo samples per user: $samples_per_user
+
+# random_seeds for which to rerun the experiments
+random_seeds=("97")
+# random_seed = 97
+echo batchsize: $batchsize
+echo communication rounds per global epoch: $comm_rounds_per_global_epoch
+# calculating how many batches there are in a global epoch for each user/proc
+batches_per_epoch=$(($samples_per_user / $batchsize))
+echo batches per global epoch: $batches_per_epoch
+# the number of iterations in 25 global epochs
+iterations=$($env_python -c "from math import floor; print($batches_per_epoch * $global_epochs) if $comm_rounds_per_global_epoch >= $batches_per_epoch else print($global_epochs * $comm_rounds_per_global_epoch)")
+echo iterations: $iterations
+# calculating the number of batches each user/proc uses per communication step (The actual number may be a float, which we round down)
+batches_per_comm_round=$($env_python -c "from math import floor; x = floor($batches_per_epoch / $comm_rounds_per_global_epoch); print(1 if x==0 else x)")
+# since the batches per communication round were rounded down we need to change the number of iterations to reflect that
+new_iterations=$($env_python -c "from math import floor; tmp = floor($batches_per_epoch / $comm_rounds_per_global_epoch); x = 1 if tmp == 0 else tmp; y = floor((($batches_per_epoch / $comm_rounds_per_global_epoch)/x)*$iterations); print($iterations if y<$iterations else y)")
+echo batches per communication round: $batches_per_comm_round
+echo corrected iterations: $new_iterations
+test_after=$(($new_iterations / $global_epochs))
+echo test after: $test_after
+for i in "${tests[@]}"
+do
+ for seed in "${random_seeds[@]}"
+ do
+ echo $i
+ IFS='_' read -ra NAMES <<< $i
+ IFS='.' read -ra NAME <<< ${NAMES[-1]}
+ log_dir_base=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')
+ echo results are stored in: $log_dir_base
+ log_dir=$log_dir_base/machine0
+ mkdir -p $log_dir
+ weight_store_dir=$log_dir_base/weights
+ mkdir -p $weight_store_dir
+ cp $i $config_file
+ # changing the config files to reflect the values of the current grid search state
+ $python_bin/crudini --set $config_file COMMUNICATION addresses_filepath $ip_machines
+ $python_bin/crudini --set $config_file OPTIMIZER_PARAMS lr $lr
+ $python_bin/crudini --set $config_file TRAIN_PARAMS rounds $batches_per_comm_round
+ $python_bin/crudini --set $config_file TRAIN_PARAMS batch_size $batchsize
+ $python_bin/crudini --set $config_file DATASET random_seed $seed
+ $env_python $eval_file -ro 0 -tea $test_after -ld $log_dir -wsd $weight_store_dir -mid 0 -ps $procs_per_machine -ms $machines -is $new_iterations -gf $graph -ta $test_after -cf $config_file -ll $log_level
+ echo $i is done
+ sleep 10
+ echo end of sleep
+ done
+done
+#
\ No newline at end of file
diff --git a/eval/step_configs/config_reddit_jwins_local.ini b/eval/step_configs/config_reddit_jwins_local.ini
new file mode 100644
index 0000000000000000000000000000000000000000..90ad9d25348a1170c1c8f12f88b7fc6b99be663d
--- /dev/null
+++ b/eval/step_configs/config_reddit_jwins_local.ini
@@ -0,0 +1,43 @@
+[DATASET]
+dataset_package = decentralizepy.datasets.Reddit
+dataset_class = Reddit
+random_seed = 97
+model_class = RNN
+train_dir = /home/jeffrey/Downloads/reddit/per_user_data/train
+test_dir = /home/jeffrey/Downloads/reddit/new_small_data/test
+; python list of fractions below
+sizes =
+
+[OPTIMIZER_PARAMS]
+optimizer_package = torch.optim
+optimizer_class = SGD
+lr = 0.001
+
+[TRAIN_PARAMS]
+training_package = decentralizepy.training.Training
+training_class = Training
+rounds = 47
+full_epochs = False
+batch_size = 16
+shuffle = True
+loss_package = torch.nn
+loss_class = CrossEntropyLoss
+
+[COMMUNICATION]
+comm_package = decentralizepy.communication.TCP
+comm_class = TCP
+addresses_filepath = ip_addr_6Machines.json
+compression_package = decentralizepy.compression.EliasFpzipLossy
+compression_class = EliasFpzipLossy
+compress = False
+
+[SHARING]
+sharing_package = decentralizepy.sharing.RandomAlphaWavelet
+sharing_class = RandomAlpha
+change_based_selection = True
+alpha_list = [0.1,0.15,0.2,0.25,0.3,0.4,1.0]
+wavelet=sym2
+level= 4
+accumulation = True
+accumulate_averaging_changes = True
+metadata_cap = 0.5
\ No newline at end of file
diff --git a/eval/step_configs/config_reddit_sharing_local2.ini b/eval/step_configs/config_reddit_sharing_local2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6a8a2eba40642f6df3d5c30cd7489554ca8bb080
--- /dev/null
+++ b/eval/step_configs/config_reddit_sharing_local2.ini
@@ -0,0 +1,36 @@
+[DATASET]
+dataset_package = decentralizepy.datasets.Reddit
+dataset_class = Reddit
+random_seed = 97
+model_class = RNN
+train_dir = /home/jeffrey/Downloads/reddit/per_user_data/train
+test_dir = /home/jeffrey/Downloads/reddit/new_small_data/test
+; python list of fractions below
+sizes =
+
+[OPTIMIZER_PARAMS]
+optimizer_package = torch.optim
+optimizer_class = SGD
+lr = 0.001
+
+[TRAIN_PARAMS]
+training_package = decentralizepy.training.Training
+training_class = Training
+rounds = 47
+full_epochs = False
+batch_size = 16
+shuffle = True
+loss_package = torch.nn
+loss_class = CrossEntropyLoss
+
+[COMMUNICATION]
+comm_package = decentralizepy.communication.TCP
+comm_class = TCP
+addresses_filepath = ip_addr_6Machines.json
+compression_package = decentralizepy.compression.Elias
+compression_class = Elias
+compress = False
+
+[SHARING]
+sharing_package = decentralizepy.sharing.Sharing
+sharing_class = Sharing
diff --git a/eval/step_configs/config_reddit_topkacc_local.ini b/eval/step_configs/config_reddit_topkacc_local.ini
new file mode 100644
index 0000000000000000000000000000000000000000..8841ba40be85af8afab59dcc9958b3b787ac577b
--- /dev/null
+++ b/eval/step_configs/config_reddit_topkacc_local.ini
@@ -0,0 +1,51 @@
+[DATASET]
+dataset_package = decentralizepy.datasets.Reddit
+dataset_class = Reddit
+random_seed = 97
+model_class = RNN
+train_dir = /home/jeffrey/Downloads/reddit/per_user_data/train
+test_dir = /home/jeffrey/Downloads/reddit/new_small_data/test
+; python list of fractions below
+sizes =
+
+[OPTIMIZER_PARAMS]
+optimizer_package = torch.optim
+optimizer_class = SGD
+lr = 0.001
+
+[TRAIN_PARAMS]
+training_package = decentralizepy.training.Training
+training_class = Training
+rounds = 47
+full_epochs = False
+batch_size = 16
+shuffle = True
+loss_package = torch.nn
+loss_class = CrossEntropyLoss
+
+[COMMUNICATION]
+comm_package = decentralizepy.communication.TCP
+comm_class = TCP
+addresses_filepath = ip_addr_6Machines.json
+compression_package = decentralizepy.compression.Eliaszfplossy1
+compression_class = Eliaszfplossy1
+compress = True
+
+[SHARING]
+sharing_package = decentralizepy.sharing.WaveletBound
+sharing_class = WaveletBound
+alpha=0.1
+lower_bound=0.1
+metro_hastings=True
+;sharing_package = decentralizepy.sharing.PartialModel
+;sharing_class = PartialModel
+;alpha = 0.1
+;accumulation = True
+;accumulate_averaging_changes = True
+;sharing_package = decentralizepy.sharing.RandomAlphaWavelet
+;sharing_class = RandomAlpha
+;change_based_selection = True
+;wavelet=sym2
+;level= 4
+;accumulation = True
+;accumulate_averaging_changes = True
\ No newline at end of file
diff --git a/random files/16_regular.edges b/random files/16_regular.edges
new file mode 100644
index 0000000000000000000000000000000000000000..4e847b9534add5352272851013ac3c1994f09b43
--- /dev/null
+++ b/random files/16_regular.edges
@@ -0,0 +1,49 @@
+16
+0 1
+0 3
+0 15
+1 0
+1 10
+1 2
+2 1
+2 3
+2 15
+3 0
+3 2
+3 4
+4 9
+4 3
+4 5
+5 4
+5 13
+5 6
+6 11
+6 5
+6 7
+7 8
+7 14
+7 6
+8 9
+8 12
+8 7
+9 8
+9 10
+9 4
+10 1
+10 11
+10 9
+11 10
+11 12
+11 6
+12 8
+12 11
+12 13
+13 12
+13 5
+13 14
+14 15
+14 13
+14 7
+15 0
+15 2
+15 14
diff --git a/random files/16_regular.png b/random files/16_regular.png
new file mode 100644
index 0000000000000000000000000000000000000000..d9b3acd3227cbb311a5e2da33bfb56138f52d887
Binary files /dev/null and b/random files/16_regular.png differ
diff --git a/random files/16_ring.edges b/random files/16_ring.edges
new file mode 100644
index 0000000000000000000000000000000000000000..f94b76a8a0b04c2c93def6590acf53c90da514c7
--- /dev/null
+++ b/random files/16_ring.edges
@@ -0,0 +1,33 @@
+16
+0 1
+0 15
+1 0
+1 2
+2 1
+2 3
+3 2
+3 4
+4 3
+4 5
+5 4
+5 6
+6 5
+6 7
+7 8
+7 6
+8 9
+8 7
+9 8
+9 10
+10 9
+10 11
+11 10
+11 12
+12 11
+12 13
+13 12
+13 14
+14 13
+14 15
+15 0
+15 14
diff --git a/random files/16_ring.png b/random files/16_ring.png
new file mode 100644
index 0000000000000000000000000000000000000000..2a5f1cce138624fa76a06028f8fa4f6d63d6439a
Binary files /dev/null and b/random files/16_ring.png differ
diff --git a/random files/192_regular.edges b/random files/192_regular.edges
new file mode 100644
index 0000000000000000000000000000000000000000..71162a775d3ac2f7e0ed2e0965d307b059f4e0e9
--- /dev/null
+++ b/random files/192_regular.edges
@@ -0,0 +1,961 @@
+192
+0 129
+0 33
+0 3
+0 113
+0 178
+1 98
+1 132
+1 12
+1 186
+1 59
+2 80
+2 147
+2 53
+2 55
+2 189
+3 0
+3 36
+3 109
+3 111
+3 176
+4 104
+4 42
+4 86
+4 185
+4 27
+5 21
+5 157
+5 123
+5 125
+5 63
+6 41
+6 171
+6 48
+6 58
+6 189
+7 32
+7 105
+7 171
+7 176
+7 126
+8 96
+8 65
+8 140
+8 54
+8 25
+9 91
+9 13
+9 83
+9 184
+9 27
+10 44
+10 13
+10 14
+10 79
+10 157
+11 160
+11 132
+11 142
+11 80
+11 158
+12 1
+12 134
+12 147
+12 85
+12 188
+13 66
+13 9
+13 10
+13 185
+13 127
+14 169
+14 10
+14 55
+14 92
+14 126
+15 103
+15 104
+15 170
+15 112
+15 17
+16 138
+16 141
+16 143
+16 149
+16 157
+17 35
+17 37
+17 15
+17 144
+17 190
+18 160
+18 35
+18 102
+18 186
+18 63
+19 78
+19 48
+19 183
+19 60
+19 191
+20 130
+20 133
+20 180
+20 117
+20 118
+21 164
+21 5
+21 135
+21 179
+21 117
+22 100
+22 108
+22 89
+22 60
+22 30
+23 132
+23 168
+23 170
+23 107
+23 110
+24 66
+24 45
+24 80
+24 26
+24 62
+25 96
+25 8
+25 110
+25 82
+25 182
+26 97
+26 163
+26 48
+26 24
+26 127
+27 4
+27 9
+27 78
+27 112
+27 62
+28 138
+28 107
+28 111
+28 176
+28 122
+29 36
+29 172
+29 79
+29 146
+29 89
+30 130
+30 78
+30 22
+30 87
+30 123
+31 64
+31 101
+31 172
+31 175
+31 50
+32 7
+32 73
+32 41
+32 113
+32 188
+33 0
+33 131
+33 145
+33 115
+33 116
+34 163
+34 100
+34 142
+34 56
+34 88
+35 99
+35 165
+35 17
+35 18
+35 191
+36 3
+36 37
+36 74
+36 114
+36 29
+37 129
+37 36
+37 109
+37 47
+37 17
+38 98
+38 136
+38 169
+38 111
+38 51
+39 162
+39 82
+39 83
+39 182
+39 91
+40 160
+40 99
+40 110
+40 82
+40 184
+41 32
+41 6
+41 135
+41 106
+41 94
+42 4
+42 71
+42 172
+42 145
+42 61
+43 167
+43 113
+43 181
+43 85
+43 151
+44 129
+44 166
+44 169
+44 10
+44 146
+45 130
+45 46
+45 85
+45 181
+45 24
+46 129
+46 45
+46 174
+46 180
+46 158
+47 128
+47 37
+47 141
+47 115
+47 84
+48 131
+48 6
+48 177
+48 19
+48 26
+49 177
+49 147
+49 118
+49 55
+49 58
+50 128
+50 165
+50 183
+50 56
+50 31
+51 38
+51 141
+51 80
+51 119
+51 90
+52 107
+52 175
+52 114
+52 184
+52 92
+53 129
+53 2
+53 176
+53 84
+53 63
+54 69
+54 8
+54 173
+54 181
+54 185
+55 2
+55 166
+55 14
+55 49
+55 88
+56 34
+56 78
+56 50
+56 150
+56 156
+57 69
+57 101
+57 173
+57 120
+57 122
+58 164
+58 6
+58 142
+58 49
+58 187
+59 1
+59 77
+59 174
+59 143
+59 114
+60 161
+60 71
+60 167
+60 19
+60 22
+61 70
+61 72
+61 42
+61 75
+61 77
+62 180
+62 24
+62 27
+62 125
+62 63
+63 65
+63 5
+63 18
+63 53
+63 62
+64 98
+64 163
+64 144
+64 89
+64 31
+65 100
+65 8
+65 148
+65 182
+65 63
+66 132
+66 68
+66 13
+66 87
+66 24
+67 133
+67 137
+67 76
+67 118
+67 154
+68 128
+68 66
+68 164
+68 171
+68 117
+69 171
+69 172
+69 85
+69 54
+69 57
+70 131
+70 79
+70 81
+70 186
+70 61
+71 131
+71 134
+71 42
+71 138
+71 60
+72 133
+72 104
+72 175
+72 145
+72 61
+73 32
+73 103
+73 145
+73 116
+73 153
+74 99
+74 36
+74 183
+74 155
+74 159
+75 165
+75 143
+75 185
+75 61
+75 158
+76 96
+76 67
+76 174
+76 178
+76 120
+77 103
+77 141
+77 90
+77 59
+77 61
+78 139
+78 19
+78 56
+78 27
+78 30
+79 162
+79 70
+79 10
+79 120
+79 29
+80 2
+80 11
+80 51
+80 24
+80 190
+81 70
+81 134
+81 141
+81 153
+81 189
+82 39
+82 40
+82 137
+82 135
+82 25
+83 39
+83 9
+83 86
+83 154
+83 94
+84 102
+84 105
+84 47
+84 53
+84 121
+85 69
+85 104
+85 43
+85 12
+85 45
+86 4
+86 133
+86 144
+86 83
+86 180
+87 66
+87 144
+87 146
+87 30
+87 159
+88 128
+88 34
+88 139
+88 55
+88 91
+89 64
+89 22
+89 151
+89 184
+89 29
+90 77
+90 173
+90 51
+90 156
+90 190
+91 39
+91 9
+91 151
+91 88
+91 188
+92 121
+92 131
+92 14
+92 52
+92 185
+93 161
+93 168
+93 143
+93 176
+93 182
+94 162
+94 166
+94 167
+94 41
+94 83
+95 130
+95 163
+95 105
+95 111
+95 117
+96 8
+96 139
+96 76
+96 25
+96 155
+97 162
+97 137
+97 147
+97 181
+97 26
+98 64
+98 1
+98 38
+98 152
+98 158
+99 35
+99 164
+99 40
+99 74
+99 142
+100 65
+100 34
+100 179
+100 22
+100 189
+101 110
+101 148
+101 57
+101 124
+101 31
+102 106
+102 114
+102 18
+102 148
+102 84
+103 104
+103 73
+103 109
+103 77
+103 15
+104 4
+104 103
+104 72
+104 15
+104 85
+105 7
+105 106
+105 177
+105 84
+105 95
+106 102
+106 41
+106 105
+106 172
+106 150
+107 23
+107 52
+107 183
+107 28
+107 191
+108 130
+108 134
+108 22
+108 120
+108 155
+109 3
+109 37
+109 103
+109 173
+109 149
+110 101
+110 40
+110 23
+110 25
+110 126
+111 3
+111 38
+111 116
+111 28
+111 95
+112 155
+112 174
+112 15
+112 27
+112 189
+113 160
+113 0
+113 32
+113 43
+113 115
+114 36
+114 102
+114 52
+114 150
+114 59
+115 33
+115 168
+115 136
+115 47
+115 113
+116 33
+116 73
+116 170
+116 111
+116 127
+117 68
+117 20
+117 21
+117 154
+117 95
+118 67
+118 140
+118 49
+118 20
+118 152
+119 174
+119 51
+119 149
+119 120
+119 187
+120 108
+120 76
+120 79
+120 119
+120 57
+121 162
+121 165
+121 170
+121 84
+121 92
+122 28
+122 180
+122 57
+122 187
+122 124
+123 5
+123 137
+123 30
+123 183
+123 190
+124 161
+124 101
+124 154
+124 140
+124 122
+125 5
+125 136
+125 177
+125 184
+125 62
+126 7
+126 173
+126 110
+126 175
+126 14
+127 135
+127 167
+127 13
+127 116
+127 26
+128 68
+128 47
+128 50
+128 88
+128 157
+129 0
+129 37
+129 44
+129 46
+129 53
+130 108
+130 45
+130 20
+130 30
+130 95
+131 33
+131 70
+131 71
+131 48
+131 92
+132 1
+132 66
+132 11
+132 23
+132 156
+133 67
+133 72
+133 148
+133 20
+133 86
+134 166
+134 71
+134 108
+134 12
+134 81
+135 41
+135 178
+135 82
+135 21
+135 127
+136 165
+136 38
+136 115
+136 125
+136 159
+137 97
+137 161
+137 67
+137 82
+137 123
+138 71
+138 16
+138 146
+138 182
+138 28
+139 96
+139 78
+139 150
+139 88
+139 153
+140 8
+140 124
+140 118
+140 153
+140 188
+141 77
+141 47
+141 16
+141 81
+141 51
+142 34
+142 99
+142 11
+142 151
+142 58
+143 75
+143 16
+143 186
+143 59
+143 93
+144 64
+144 171
+144 17
+144 86
+144 87
+145 33
+145 72
+145 73
+145 42
+145 153
+146 138
+146 44
+146 178
+146 87
+146 29
+147 97
+147 2
+147 166
+147 12
+147 49
+148 65
+148 133
+148 101
+148 102
+148 191
+149 164
+149 168
+149 109
+149 16
+149 119
+150 106
+150 139
+150 175
+150 114
+150 56
+151 168
+151 43
+151 142
+151 89
+151 91
+152 161
+152 98
+152 118
+152 154
+152 187
+153 73
+153 139
+153 140
+153 81
+153 145
+154 67
+154 83
+154 117
+154 152
+154 124
+155 96
+155 74
+155 108
+155 112
+155 190
+156 132
+156 169
+156 181
+156 56
+156 90
+157 128
+157 163
+157 5
+157 10
+157 16
+158 98
+158 75
+158 11
+158 46
+158 178
+159 160
+159 136
+159 74
+159 179
+159 87
+160 40
+160 11
+160 113
+160 18
+160 159
+161 137
+161 60
+161 152
+161 124
+161 93
+162 97
+162 39
+162 79
+162 121
+162 94
+163 64
+163 34
+163 26
+163 157
+163 95
+164 99
+164 68
+164 21
+164 149
+164 58
+165 35
+165 136
+165 75
+165 50
+165 121
+166 134
+166 44
+166 147
+166 55
+166 94
+167 170
+167 43
+167 60
+167 94
+167 127
+168 115
+168 23
+168 149
+168 151
+168 93
+169 38
+169 44
+169 14
+169 179
+169 156
+170 167
+170 15
+170 116
+170 23
+170 121
+171 68
+171 69
+171 6
+171 7
+171 144
+172 69
+172 42
+172 106
+172 29
+172 31
+173 109
+173 54
+173 57
+173 90
+173 126
+174 76
+174 46
+174 112
+174 119
+174 59
+175 72
+175 52
+175 150
+175 126
+175 31
+176 3
+176 7
+176 53
+176 28
+176 93
+177 105
+177 48
+177 49
+177 187
+177 125
+178 0
+178 135
+178 76
+178 146
+178 158
+179 100
+179 169
+179 21
+179 186
+179 159
+180 46
+180 20
+180 86
+180 122
+180 62
+181 97
+181 43
+181 45
+181 54
+181 156
+182 65
+182 39
+182 138
+182 25
+182 93
+183 74
+183 107
+183 50
+183 19
+183 123
+184 40
+184 9
+184 52
+184 89
+184 125
+185 4
+185 75
+185 13
+185 54
+185 92
+186 1
+186 70
+186 143
+186 18
+186 179
+187 122
+187 177
+187 119
+187 152
+187 58
+188 32
+188 140
+188 12
+188 91
+188 191
+189 2
+189 100
+189 6
+189 112
+189 81
+190 155
+190 80
+190 17
+190 90
+190 123
+191 35
+191 107
+191 19
+191 148
+191 188
diff --git a/random files/192_regular.png b/random files/192_regular.png
new file mode 100644
index 0000000000000000000000000000000000000000..ef86b2e34344d2b04e0df33d78ea4fb8c0cd1154
Binary files /dev/null and b/random files/192_regular.png differ
diff --git a/random files/288_regular.edges b/random files/288_regular.edges
new file mode 100644
index 0000000000000000000000000000000000000000..3c7c269b0483ff854d31272c6e556d1f22407327
--- /dev/null
+++ b/random files/288_regular.edges
@@ -0,0 +1,1441 @@
+288
+0 193
+0 196
+0 44
+0 50
+0 185
+1 164
+1 136
+1 244
+1 183
+1 63
+2 260
+2 46
+2 273
+2 51
+2 25
+3 259
+3 197
+3 39
+3 232
+3 60
+4 226
+4 268
+4 238
+4 284
+4 188
+5 43
+5 141
+5 240
+5 81
+5 211
+6 97
+6 101
+6 7
+6 76
+6 152
+7 33
+7 6
+7 38
+7 9
+7 287
+8 236
+8 241
+8 178
+8 252
+8 220
+9 7
+9 77
+9 272
+9 211
+9 190
+10 34
+10 266
+10 172
+10 175
+10 241
+11 165
+11 12
+11 210
+11 115
+11 219
+12 131
+12 11
+12 176
+12 183
+12 94
+13 212
+13 278
+13 217
+13 187
+13 223
+14 261
+14 106
+14 141
+14 47
+14 280
+15 258
+15 110
+15 208
+15 119
+15 30
+16 139
+16 173
+16 83
+16 59
+16 125
+17 169
+17 50
+17 18
+17 212
+17 180
+18 163
+18 108
+18 17
+18 242
+18 281
+19 66
+19 26
+19 261
+19 218
+19 220
+20 160
+20 35
+20 174
+20 270
+20 253
+21 171
+21 205
+21 51
+21 55
+21 185
+22 264
+22 268
+22 117
+22 23
+22 91
+23 201
+23 271
+23 22
+23 190
+23 94
+24 163
+24 69
+24 168
+24 139
+24 215
+25 65
+25 2
+25 123
+25 125
+25 127
+26 162
+26 45
+26 177
+26 113
+26 19
+27 225
+27 98
+27 163
+27 48
+27 146
+28 160
+28 100
+28 164
+28 133
+28 85
+29 64
+29 229
+29 199
+29 211
+29 221
+30 39
+30 270
+30 15
+30 273
+30 189
+31 164
+31 166
+31 199
+31 103
+31 46
+32 64
+32 256
+32 104
+32 144
+32 156
+33 7
+33 266
+33 186
+33 221
+33 158
+34 230
+34 10
+34 115
+34 148
+34 286
+35 160
+35 168
+35 20
+35 91
+35 222
+36 133
+36 198
+36 239
+36 146
+36 218
+37 162
+37 92
+37 153
+37 123
+37 188
+38 198
+38 7
+38 76
+38 206
+38 120
+39 3
+39 101
+39 266
+39 282
+39 30
+40 194
+40 71
+40 111
+40 239
+40 148
+41 105
+41 78
+41 250
+41 253
+41 190
+42 239
+42 114
+42 115
+42 214
+42 218
+43 5
+43 209
+43 246
+43 247
+43 94
+44 0
+44 234
+44 174
+44 117
+44 124
+45 213
+45 117
+45 247
+45 26
+45 62
+46 161
+46 2
+46 228
+46 252
+46 31
+47 197
+47 266
+47 14
+47 282
+47 251
+48 192
+48 67
+48 195
+48 143
+48 27
+49 116
+49 52
+49 182
+49 189
+49 94
+50 0
+50 259
+50 265
+50 269
+50 17
+51 2
+51 163
+51 70
+51 211
+51 21
+52 258
+52 68
+52 140
+52 49
+52 54
+53 200
+53 137
+53 108
+53 151
+53 56
+54 75
+54 208
+54 52
+54 221
+54 61
+55 73
+55 141
+55 148
+55 21
+55 189
+56 101
+56 233
+56 239
+56 53
+56 287
+57 136
+57 148
+57 218
+57 91
+57 156
+58 216
+58 89
+58 88
+58 156
+58 93
+59 256
+59 197
+59 173
+59 16
+59 149
+60 3
+60 103
+60 83
+60 214
+60 157
+61 230
+61 264
+61 243
+61 54
+61 152
+62 224
+62 65
+62 45
+62 173
+62 84
+63 1
+63 228
+63 105
+63 75
+63 271
+64 32
+64 137
+64 147
+64 121
+64 29
+65 76
+65 276
+65 25
+65 62
+65 191
+66 167
+66 19
+66 275
+66 118
+66 126
+67 92
+67 48
+67 121
+67 124
+67 125
+68 200
+68 204
+68 52
+68 118
+68 156
+69 235
+69 215
+69 24
+69 249
+69 159
+70 202
+70 267
+70 106
+70 240
+70 51
+71 226
+71 135
+71 40
+71 167
+71 241
+72 161
+72 231
+72 151
+72 280
+72 287
+73 258
+73 269
+73 207
+73 55
+73 90
+74 174
+74 209
+74 210
+74 119
+74 284
+75 166
+75 79
+75 54
+75 279
+75 63
+76 65
+76 227
+76 6
+76 38
+76 215
+77 231
+77 9
+77 183
+77 154
+77 127
+78 224
+78 100
+78 41
+78 157
+78 223
+79 262
+79 104
+79 75
+79 206
+79 215
+80 257
+80 260
+80 236
+80 277
+80 154
+81 224
+81 194
+81 5
+81 230
+81 282
+82 265
+82 147
+82 150
+82 280
+82 90
+83 225
+83 16
+83 184
+83 122
+83 60
+84 99
+84 262
+84 105
+84 120
+84 62
+85 244
+85 151
+85 121
+85 122
+85 28
+86 226
+86 233
+86 201
+86 207
+86 120
+87 192
+87 199
+87 146
+87 123
+87 222
+88 106
+88 144
+88 114
+88 275
+88 58
+89 99
+89 180
+89 276
+89 58
+89 223
+90 73
+90 177
+90 82
+90 186
+90 126
+91 35
+91 140
+91 173
+91 22
+91 57
+92 67
+92 99
+92 37
+92 274
+92 283
+93 263
+93 277
+93 58
+93 253
+93 191
+94 195
+94 43
+94 12
+94 49
+94 23
+95 132
+95 103
+95 140
+95 179
+95 149
+96 188
+96 116
+96 281
+96 251
+96 124
+97 166
+97 167
+97 6
+97 233
+97 172
+98 128
+98 233
+98 279
+98 185
+98 27
+99 242
+99 84
+99 246
+99 89
+99 92
+100 78
+100 272
+100 183
+100 247
+100 28
+101 6
+101 39
+101 236
+101 143
+101 56
+102 129
+102 195
+102 200
+102 264
+102 178
+103 137
+103 31
+103 60
+103 95
+103 159
+104 224
+104 32
+104 235
+104 79
+104 220
+105 41
+105 108
+105 84
+105 63
+105 255
+106 70
+106 14
+106 88
+106 250
+106 157
+107 166
+107 202
+107 270
+107 216
+107 121
+108 105
+108 138
+108 269
+108 18
+108 53
+109 165
+109 111
+109 275
+109 147
+109 245
+110 260
+110 235
+110 206
+110 15
+110 146
+111 259
+111 40
+111 138
+111 109
+111 254
+112 132
+112 235
+112 274
+112 178
+112 275
+113 154
+113 251
+113 217
+113 26
+113 155
+114 42
+114 118
+114 152
+114 153
+114 88
+115 160
+115 34
+115 42
+115 11
+115 236
+116 96
+116 194
+116 49
+116 122
+116 187
+117 44
+117 45
+117 207
+117 22
+117 223
+118 66
+118 196
+118 68
+118 231
+118 114
+119 74
+119 15
+119 212
+119 219
+119 127
+120 38
+120 176
+120 273
+120 84
+120 86
+121 64
+121 67
+121 107
+121 245
+121 85
+122 144
+122 83
+122 116
+122 85
+122 285
+123 128
+123 37
+123 87
+123 25
+123 158
+124 96
+124 67
+124 44
+124 208
+124 150
+125 67
+125 172
+125 16
+125 25
+125 189
+126 160
+126 66
+126 238
+126 90
+126 254
+127 203
+127 77
+127 119
+127 216
+127 25
+128 98
+128 139
+128 175
+128 248
+128 123
+129 132
+129 102
+129 264
+129 169
+129 158
+130 167
+130 237
+130 178
+130 217
+130 154
+131 133
+131 138
+131 203
+131 12
+131 147
+132 129
+132 199
+132 171
+132 112
+132 95
+133 131
+133 36
+133 219
+133 28
+133 221
+134 232
+134 137
+134 208
+134 278
+134 219
+135 71
+135 263
+135 204
+135 282
+135 284
+136 193
+136 1
+136 237
+136 57
+136 286
+137 64
+137 134
+137 103
+137 53
+137 182
+138 131
+138 202
+138 108
+138 111
+138 191
+139 128
+139 16
+139 212
+139 24
+139 189
+140 159
+140 52
+140 185
+140 91
+140 95
+141 5
+141 14
+141 55
+141 154
+141 191
+142 237
+142 238
+142 185
+142 221
+142 158
+143 257
+143 101
+143 232
+143 268
+143 48
+144 32
+144 225
+144 88
+144 249
+144 122
+145 164
+145 197
+145 229
+145 283
+145 188
+146 36
+146 110
+146 277
+146 87
+146 27
+147 64
+147 258
+147 131
+147 109
+147 82
+148 34
+148 40
+148 55
+148 57
+148 253
+149 228
+149 173
+149 246
+149 59
+149 95
+150 234
+150 82
+150 184
+150 186
+150 124
+151 72
+151 85
+151 53
+151 182
+151 155
+152 6
+152 114
+152 184
+152 61
+152 191
+153 37
+153 269
+153 210
+153 242
+153 114
+154 130
+154 77
+154 141
+154 80
+154 113
+155 261
+155 172
+155 113
+155 151
+155 286
+156 32
+156 68
+156 176
+156 57
+156 58
+157 106
+157 268
+157 78
+157 282
+157 60
+158 129
+158 33
+158 142
+158 177
+158 123
+159 69
+159 103
+159 268
+159 140
+159 209
+160 35
+160 115
+160 20
+160 28
+160 126
+161 163
+161 167
+161 72
+161 46
+161 219
+162 37
+162 232
+162 205
+162 26
+162 255
+163 161
+163 18
+163 51
+163 24
+163 27
+164 1
+164 271
+164 145
+164 28
+164 31
+165 225
+165 11
+165 109
+165 244
+165 182
+166 97
+166 266
+166 75
+166 107
+166 31
+167 161
+167 66
+167 130
+167 97
+167 71
+168 35
+168 238
+168 180
+168 213
+168 24
+169 129
+169 195
+169 17
+169 179
+169 243
+170 261
+170 197
+170 213
+170 248
+170 285
+171 256
+171 227
+171 132
+171 21
+171 181
+172 97
+172 10
+172 179
+172 155
+172 125
+173 91
+173 16
+173 149
+173 59
+173 62
+174 234
+174 74
+174 44
+174 20
+174 213
+175 128
+175 10
+175 206
+175 177
+175 190
+176 232
+176 12
+176 273
+176 120
+176 156
+177 90
+177 202
+177 175
+177 26
+177 158
+178 130
+178 102
+178 8
+178 207
+178 112
+179 169
+179 172
+179 210
+179 279
+179 95
+180 168
+180 205
+180 17
+180 181
+180 89
+181 196
+181 265
+181 171
+181 271
+181 180
+182 165
+182 137
+182 208
+182 49
+182 151
+183 1
+183 100
+183 12
+183 77
+183 244
+184 280
+184 240
+184 83
+184 150
+184 152
+185 0
+185 98
+185 140
+185 142
+185 21
+186 33
+186 245
+186 150
+186 90
+186 287
+187 13
+187 274
+187 243
+187 116
+187 216
+188 96
+188 4
+188 37
+188 145
+188 242
+189 139
+189 49
+189 55
+189 125
+189 30
+190 41
+190 9
+190 175
+190 23
+190 222
+191 65
+191 138
+191 141
+191 152
+191 93
+192 263
+192 48
+192 276
+192 87
+192 285
+193 0
+193 194
+193 262
+193 136
+193 252
+194 193
+194 195
+194 40
+194 81
+194 116
+195 194
+195 102
+195 169
+195 48
+195 94
+196 0
+196 204
+196 181
+196 118
+196 251
+197 3
+197 170
+197 47
+197 145
+197 59
+198 36
+198 38
+198 205
+198 245
+198 255
+199 132
+199 214
+199 87
+199 29
+199 31
+200 68
+200 102
+200 267
+200 272
+200 53
+201 227
+201 204
+201 274
+201 86
+201 23
+202 70
+202 138
+202 107
+202 177
+202 247
+203 131
+203 277
+203 284
+203 222
+203 127
+204 196
+204 68
+204 135
+204 201
+204 237
+205 257
+205 162
+205 198
+205 180
+205 21
+206 38
+206 262
+206 110
+206 79
+206 175
+207 73
+207 238
+207 178
+207 117
+207 86
+208 134
+208 15
+208 182
+208 54
+208 124
+209 74
+209 43
+209 239
+209 277
+209 159
+210 74
+210 11
+210 179
+210 246
+210 153
+211 5
+211 9
+211 51
+211 29
+211 255
+212 139
+212 13
+212 17
+212 278
+212 119
+213 168
+213 170
+213 45
+213 174
+213 245
+214 199
+214 42
+214 267
+214 60
+214 255
+215 69
+215 76
+215 79
+215 24
+215 223
+216 107
+216 279
+216 58
+216 187
+216 127
+217 130
+217 228
+217 13
+217 113
+217 244
+218 36
+218 42
+218 19
+218 276
+218 57
+219 161
+219 133
+219 134
+219 11
+219 119
+220 8
+220 104
+220 19
+220 278
+220 253
+221 33
+221 133
+221 142
+221 54
+221 29
+222 35
+222 203
+222 87
+222 251
+222 190
+223 13
+223 78
+223 117
+223 215
+223 89
+224 104
+224 269
+224 78
+224 81
+224 62
+225 165
+225 144
+225 83
+225 250
+225 27
+226 227
+226 4
+226 71
+226 86
+226 283
+227 226
+227 201
+227 234
+227 171
+227 76
+228 217
+228 46
+228 149
+228 281
+228 63
+229 256
+229 258
+229 231
+229 145
+229 29
+230 34
+230 259
+230 231
+230 81
+230 61
+231 229
+231 230
+231 72
+231 77
+231 118
+232 162
+232 3
+232 134
+232 143
+232 176
+233 97
+233 98
+233 86
+233 279
+233 56
+234 227
+234 44
+234 174
+234 150
+234 252
+235 69
+235 104
+235 110
+235 270
+235 112
+236 101
+236 8
+236 80
+236 115
+236 280
+237 130
+237 136
+237 204
+237 142
+237 272
+238 4
+238 168
+238 142
+238 207
+238 126
+239 36
+239 40
+239 42
+239 209
+239 56
+240 257
+240 5
+240 70
+240 241
+240 184
+241 71
+241 8
+241 10
+241 240
+241 278
+242 99
+242 18
+242 153
+242 283
+242 188
+243 169
+243 265
+243 187
+243 61
+243 254
+244 1
+244 165
+244 85
+244 183
+244 217
+245 198
+245 109
+245 213
+245 121
+245 186
+246 99
+246 43
+246 210
+246 149
+246 286
+247 100
+247 202
+247 43
+247 45
+247 284
+248 128
+248 170
+248 272
+248 254
+248 287
+249 259
+249 69
+249 144
+249 273
+249 281
+250 225
+250 264
+250 41
+250 106
+250 271
+251 96
+251 196
+251 47
+251 113
+251 222
+252 193
+252 8
+252 234
+252 46
+252 254
+253 41
+253 20
+253 148
+253 220
+253 93
+254 111
+254 243
+254 248
+254 252
+254 126
+255 162
+255 198
+255 105
+255 211
+255 214
+256 32
+256 261
+256 229
+256 171
+256 59
+257 260
+257 205
+257 143
+257 80
+257 240
+258 229
+258 73
+258 15
+258 147
+258 52
+259 3
+259 230
+259 111
+259 50
+259 249
+260 257
+260 2
+260 263
+260 110
+260 80
+261 256
+261 170
+261 14
+261 19
+261 155
+262 193
+262 206
+262 79
+262 84
+262 285
+263 192
+263 260
+263 135
+263 281
+263 93
+264 129
+264 102
+264 22
+264 250
+264 61
+265 267
+265 82
+265 243
+265 50
+265 181
+266 33
+266 166
+266 39
+266 10
+266 47
+267 70
+267 200
+267 265
+267 276
+267 214
+268 4
+268 143
+268 22
+268 157
+268 159
+269 224
+269 73
+269 108
+269 50
+269 153
+270 107
+270 235
+270 20
+270 285
+270 30
+271 164
+271 181
+271 23
+271 250
+271 63
+272 100
+272 200
+272 9
+272 237
+272 248
+273 2
+273 176
+273 120
+273 249
+273 30
+274 201
+274 187
+274 112
+274 283
+274 92
+275 66
+275 109
+275 112
+275 88
+275 286
+276 192
+276 65
+276 267
+276 89
+276 218
+277 203
+277 80
+277 209
+277 146
+277 93
+278 134
+278 13
+278 241
+278 212
+278 220
+279 98
+279 233
+279 75
+279 179
+279 216
+280 72
+280 236
+280 14
+280 82
+280 184
+281 96
+281 228
+281 263
+281 18
+281 249
+282 135
+282 39
+282 47
+282 81
+282 157
+283 226
+283 145
+283 274
+283 242
+283 92
+284 4
+284 135
+284 74
+284 203
+284 247
+285 192
+285 262
+285 170
+285 270
+285 122
+286 34
+286 136
+286 275
+286 246
+286 155
+287 7
+287 186
+287 72
+287 248
+287 56
diff --git a/random files/384_regular.edges b/random files/384_regular.edges
new file mode 100644
index 0000000000000000000000000000000000000000..14b37745a27a4a1e04d6210169b296f5846569bc
--- /dev/null
+++ b/random files/384_regular.edges
@@ -0,0 +1,2305 @@
+384
+0 42
+0 203
+0 174
+0 212
+0 180
+0 22
+1 271
+1 304
+1 272
+1 212
+1 25
+1 282
+2 256
+2 354
+2 21
+2 54
+2 152
+2 313
+3 268
+3 332
+3 211
+3 20
+3 150
+3 119
+4 10
+4 42
+4 332
+4 247
+4 91
+4 254
+5 199
+5 204
+5 237
+5 374
+5 376
+5 157
+6 352
+6 296
+6 379
+6 21
+6 280
+6 347
+7 288
+7 194
+7 293
+7 174
+7 54
+7 310
+8 103
+8 333
+8 143
+8 242
+8 50
+8 87
+9 41
+9 203
+9 301
+9 112
+9 180
+9 53
+10 130
+10 4
+10 172
+10 141
+10 334
+10 22
+11 265
+11 43
+11 79
+11 341
+11 245
+11 283
+12 67
+12 73
+12 123
+12 27
+12 124
+12 253
+13 160
+13 297
+13 171
+13 173
+13 308
+13 248
+14 358
+14 294
+14 302
+14 339
+14 24
+14 286
+15 129
+15 361
+15 309
+15 153
+15 380
+15 94
+16 199
+16 170
+16 77
+16 240
+16 180
+16 319
+17 65
+17 68
+17 324
+17 43
+17 211
+17 245
+18 324
+18 231
+18 140
+18 305
+18 86
+18 183
+19 200
+19 363
+19 83
+19 211
+19 310
+19 189
+20 192
+20 3
+20 341
+20 278
+20 314
+20 59
+21 2
+21 163
+21 197
+21 6
+21 348
+21 63
+22 0
+22 33
+22 229
+22 10
+22 142
+22 115
+23 322
+23 131
+23 290
+23 103
+23 171
+23 81
+24 226
+24 98
+24 110
+24 14
+24 305
+24 383
+25 1
+25 314
+25 206
+25 175
+25 176
+25 90
+26 167
+26 41
+26 273
+26 340
+26 379
+26 188
+27 34
+27 138
+27 363
+27 12
+27 317
+27 255
+28 320
+28 293
+28 234
+28 209
+28 210
+28 91
+29 325
+29 267
+29 151
+29 281
+29 220
+29 62
+30 193
+30 197
+30 166
+30 134
+30 46
+30 312
+31 225
+31 365
+31 109
+31 311
+31 216
+31 221
+32 96
+32 192
+32 266
+32 343
+32 156
+32 381
+33 290
+33 35
+33 73
+33 43
+33 22
+33 319
+34 140
+34 366
+34 308
+34 183
+34 27
+34 62
+35 33
+35 333
+35 276
+35 215
+35 344
+35 314
+36 258
+36 336
+36 82
+36 254
+36 62
+36 127
+37 195
+37 43
+37 175
+37 337
+37 127
+37 319
+38 353
+38 264
+38 137
+38 305
+38 52
+38 380
+39 352
+39 268
+39 371
+39 181
+39 281
+39 252
+40 197
+40 168
+40 369
+40 243
+40 277
+40 315
+41 353
+41 257
+41 26
+41 9
+41 372
+41 186
+42 0
+42 323
+42 4
+42 82
+42 309
+42 94
+43 33
+43 37
+43 11
+43 303
+43 17
+43 350
+44 288
+44 261
+44 72
+44 367
+44 372
+44 119
+45 224
+45 198
+45 136
+45 143
+45 373
+45 158
+46 382
+46 60
+46 213
+46 156
+46 30
+46 351
+47 194
+47 259
+47 101
+47 294
+47 302
+47 272
+48 96
+48 297
+48 265
+48 153
+48 155
+48 61
+49 66
+49 205
+49 141
+49 113
+49 53
+49 149
+50 8
+50 201
+50 240
+50 117
+50 92
+50 95
+51 256
+51 265
+51 81
+51 147
+51 183
+51 121
+52 131
+52 38
+52 141
+52 272
+52 115
+52 315
+53 9
+53 203
+53 365
+53 49
+53 178
+53 91
+54 129
+54 2
+54 100
+54 7
+54 154
+54 283
+55 227
+55 131
+55 72
+55 138
+55 204
+55 188
+56 103
+56 266
+56 107
+56 171
+56 109
+56 371
+57 232
+57 108
+57 212
+57 217
+57 378
+57 316
+58 69
+58 209
+58 242
+58 221
+58 94
+58 223
+59 258
+59 361
+59 239
+59 147
+59 20
+59 188
+60 261
+60 205
+60 46
+60 271
+60 147
+60 222
+61 325
+61 167
+61 363
+61 48
+61 308
+61 281
+62 34
+62 36
+62 334
+62 304
+62 377
+62 29
+63 100
+63 176
+63 21
+63 309
+63 285
+63 253
+64 164
+64 334
+64 280
+64 250
+64 219
+64 94
+65 224
+65 172
+65 305
+65 17
+65 92
+65 157
+66 97
+66 137
+66 49
+66 180
+66 345
+66 282
+67 170
+67 12
+67 237
+67 84
+67 344
+67 93
+68 323
+68 163
+68 17
+68 86
+68 379
+68 124
+69 97
+69 202
+69 210
+69 147
+69 58
+69 188
+70 231
+70 274
+70 248
+70 220
+70 350
+70 286
+71 230
+71 102
+71 326
+71 330
+71 216
+71 287
+72 96
+72 321
+72 167
+72 44
+72 55
+72 351
+73 96
+73 33
+73 101
+73 12
+73 149
+73 377
+74 102
+74 295
+74 172
+74 183
+74 280
+74 120
+75 321
+75 165
+75 246
+75 278
+75 376
+75 126
+76 348
+76 158
+76 252
+76 253
+76 190
+76 127
+77 224
+77 132
+77 368
+77 16
+77 114
+77 181
+78 322
+78 236
+78 206
+78 366
+78 377
+78 156
+79 231
+79 328
+79 11
+79 277
+79 85
+79 282
+80 290
+80 135
+80 137
+80 238
+80 336
+80 343
+81 359
+81 110
+81 146
+81 51
+81 23
+81 279
+82 195
+82 36
+82 42
+82 274
+82 344
+82 250
+83 196
+83 232
+83 337
+83 19
+83 281
+83 382
+84 67
+84 166
+84 358
+84 335
+84 346
+84 255
+85 322
+85 79
+85 213
+85 218
+85 347
+85 189
+86 100
+86 68
+86 172
+86 18
+86 311
+86 92
+87 326
+87 8
+87 235
+87 112
+87 150
+87 90
+88 97
+88 268
+88 178
+88 146
+88 217
+88 187
+89 288
+89 97
+89 167
+89 107
+89 248
+89 286
+90 164
+90 87
+90 215
+90 120
+90 25
+90 285
+91 321
+91 4
+91 145
+91 53
+91 218
+91 28
+92 65
+92 50
+92 339
+92 86
+92 122
+92 381
+93 322
+93 67
+93 330
+93 206
+93 306
+93 249
+94 64
+94 295
+94 169
+94 42
+94 15
+94 58
+95 356
+95 263
+95 105
+95 50
+95 275
+95 249
+96 32
+96 72
+96 73
+96 48
+96 253
+96 159
+97 66
+97 324
+97 69
+97 367
+97 88
+97 89
+98 135
+98 168
+98 371
+98 24
+98 313
+98 158
+99 296
+99 298
+99 330
+99 146
+99 185
+99 122
+100 352
+100 54
+100 86
+100 282
+100 279
+100 63
+101 161
+101 327
+101 168
+101 73
+101 302
+101 47
+102 71
+102 74
+102 299
+102 204
+102 365
+102 380
+103 8
+103 241
+103 179
+103 23
+103 56
+103 154
+104 193
+104 164
+104 139
+104 115
+104 378
+104 349
+105 192
+105 166
+105 266
+105 205
+105 318
+105 95
+106 171
+106 332
+106 177
+106 244
+106 215
+106 126
+107 89
+107 260
+107 113
+107 311
+107 56
+107 377
+108 325
+108 210
+108 157
+108 57
+108 156
+108 285
+109 194
+109 355
+109 307
+109 56
+109 31
+109 383
+110 260
+110 261
+110 235
+110 241
+110 81
+110 24
+111 292
+111 138
+111 364
+111 276
+111 309
+111 188
+112 130
+112 168
+112 9
+112 87
+112 151
+112 316
+113 326
+113 107
+113 171
+113 269
+113 49
+113 219
+114 226
+114 234
+114 77
+114 208
+114 339
+114 314
+115 168
+115 104
+115 307
+115 52
+115 22
+115 119
+116 290
+116 227
+116 165
+116 357
+116 313
+116 317
+117 258
+117 166
+117 198
+117 298
+117 50
+117 221
+118 355
+118 262
+118 204
+118 173
+118 344
+118 153
+119 3
+119 44
+119 208
+119 369
+119 115
+119 341
+120 233
+120 74
+120 241
+120 342
+120 90
+120 315
+121 264
+121 232
+121 366
+121 335
+121 51
+121 190
+122 160
+122 99
+122 139
+122 381
+122 92
+122 189
+123 131
+123 329
+123 12
+123 300
+123 247
+123 313
+124 224
+124 68
+124 12
+124 333
+124 279
+124 222
+125 227
+125 358
+125 177
+125 372
+125 340
+125 351
+126 197
+126 263
+126 106
+126 75
+126 153
+126 318
+127 36
+127 37
+127 199
+127 76
+127 304
+127 287
+128 257
+128 162
+128 167
+128 206
+128 277
+128 287
+129 302
+129 15
+129 177
+129 244
+129 54
+129 316
+130 165
+130 327
+130 231
+130 10
+130 112
+130 210
+131 264
+131 297
+131 52
+131 23
+131 55
+131 123
+132 328
+132 267
+132 77
+132 375
+132 218
+132 253
+133 364
+133 334
+133 238
+133 241
+133 184
+133 189
+134 320
+134 259
+134 331
+134 367
+134 375
+134 30
+135 98
+135 293
+135 331
+135 80
+135 182
+135 383
+136 225
+136 45
+136 242
+136 243
+136 345
+136 223
+137 66
+137 291
+137 38
+137 80
+137 214
+137 221
+138 201
+138 266
+138 251
+138 111
+138 55
+138 27
+139 358
+139 104
+139 360
+139 272
+139 345
+139 122
+140 289
+140 34
+140 208
+140 18
+140 216
+140 254
+141 353
+141 327
+141 10
+141 49
+141 52
+141 213
+142 301
+142 365
+142 175
+142 277
+142 22
+142 278
+143 225
+143 8
+143 45
+143 207
+143 240
+143 351
+144 261
+144 333
+144 334
+144 369
+144 373
+144 222
+145 322
+145 170
+145 338
+145 154
+145 91
+145 220
+146 352
+146 99
+146 81
+146 306
+146 88
+146 221
+147 69
+147 267
+147 51
+147 216
+147 59
+147 60
+148 228
+148 269
+148 370
+148 212
+148 375
+148 159
+149 290
+149 197
+149 73
+149 297
+149 49
+149 283
+150 3
+150 326
+150 211
+150 87
+150 279
+150 314
+151 112
+151 274
+151 340
+151 374
+151 283
+151 29
+152 2
+152 196
+152 198
+152 264
+152 188
+152 350
+153 228
+153 15
+153 303
+153 48
+153 118
+153 126
+154 289
+154 103
+154 169
+154 145
+154 273
+154 54
+155 291
+155 355
+155 48
+155 305
+155 312
+155 345
+156 32
+156 163
+156 108
+156 46
+156 78
+156 383
+157 65
+157 5
+157 361
+157 362
+157 108
+157 312
+158 98
+158 325
+158 76
+158 45
+158 367
+158 213
+159 96
+159 230
+159 178
+159 148
+159 309
+159 380
+160 122
+160 328
+160 236
+160 13
+160 179
+160 186
+161 162
+161 101
+161 230
+161 370
+161 275
+161 246
+162 128
+162 161
+162 196
+162 181
+162 187
+162 220
+163 354
+163 68
+163 204
+163 370
+163 21
+163 156
+164 64
+164 104
+164 337
+164 90
+164 251
+164 254
+165 289
+165 130
+165 356
+165 75
+165 174
+165 116
+166 329
+166 105
+166 297
+166 84
+166 117
+166 30
+167 128
+167 72
+167 247
+167 89
+167 26
+167 61
+168 256
+168 98
+168 101
+168 40
+168 112
+168 115
+169 289
+169 294
+169 331
+169 94
+169 154
+169 190
+170 321
+170 67
+170 358
+170 16
+170 145
+170 274
+171 106
+171 13
+171 113
+171 342
+171 23
+171 56
+172 65
+172 202
+172 74
+172 10
+172 332
+172 86
+173 320
+173 198
+173 331
+173 13
+173 118
+173 377
+174 0
+174 165
+174 7
+174 267
+174 313
+174 350
+175 37
+175 142
+175 271
+175 248
+175 25
+175 187
+176 195
+176 228
+176 292
+176 239
+176 25
+176 63
+177 129
+177 106
+177 240
+177 241
+177 375
+177 125
+178 360
+178 233
+178 329
+178 53
+178 88
+178 159
+179 160
+179 192
+179 103
+179 231
+179 214
+179 285
+180 0
+180 66
+180 9
+180 271
+180 16
+180 247
+181 162
+181 195
+181 325
+181 39
+181 77
+181 278
+182 135
+182 233
+182 299
+182 335
+182 308
+182 183
+183 34
+183 74
+183 18
+183 51
+183 182
+183 184
+184 133
+184 295
+184 300
+184 367
+184 183
+184 347
+185 99
+185 355
+185 196
+185 230
+185 349
+185 222
+186 160
+186 262
+186 198
+186 264
+186 41
+186 270
+187 162
+187 261
+187 327
+187 234
+187 175
+187 88
+188 69
+188 111
+188 55
+188 152
+188 26
+188 59
+189 133
+189 199
+189 250
+189 19
+189 85
+189 122
+190 352
+190 169
+190 76
+190 121
+190 346
+190 283
+191 323
+191 357
+191 296
+191 336
+191 374
+191 286
+192 32
+192 105
+192 269
+192 179
+192 20
+192 348
+193 259
+193 104
+193 363
+193 245
+193 30
+193 254
+194 321
+194 7
+194 236
+194 109
+194 270
+194 47
+195 288
+195 37
+195 239
+195 176
+195 82
+195 181
+196 162
+196 293
+196 298
+196 83
+196 152
+196 185
+197 40
+197 208
+197 21
+197 149
+197 126
+197 30
+198 45
+198 173
+198 117
+198 152
+198 186
+198 284
+199 258
+199 5
+199 16
+199 376
+199 189
+199 127
+200 227
+200 236
+200 365
+200 302
+200 19
+200 280
+201 138
+201 298
+201 300
+201 50
+201 214
+201 378
+202 69
+202 299
+202 172
+202 237
+202 301
+202 243
+203 0
+203 9
+203 242
+203 53
+203 246
+203 377
+204 163
+204 5
+204 102
+204 267
+204 118
+204 55
+205 105
+205 60
+205 335
+205 337
+205 49
+205 252
+206 128
+206 78
+206 312
+206 25
+206 93
+206 223
+207 262
+207 235
+207 143
+207 304
+207 273
+207 381
+208 197
+208 140
+208 114
+208 374
+208 119
+208 255
+209 304
+209 370
+209 343
+209 58
+209 379
+209 28
+210 130
+210 69
+210 108
+210 376
+210 345
+210 28
+211 3
+211 17
+211 19
+211 150
+211 248
+211 382
+212 0
+212 1
+212 148
+212 214
+212 376
+212 57
+213 268
+213 141
+213 46
+213 271
+213 85
+213 158
+214 324
+214 137
+214 201
+214 179
+214 212
+214 342
+215 35
+215 356
+215 90
+215 106
+215 242
+215 314
+216 292
+216 71
+216 140
+216 307
+216 147
+216 31
+217 322
+217 359
+217 235
+217 88
+217 57
+217 250
+218 289
+218 132
+218 269
+218 85
+218 91
+218 287
+219 64
+219 257
+219 354
+219 113
+219 277
+219 286
+220 162
+220 70
+220 332
+220 145
+220 243
+220 29
+221 137
+221 146
+221 276
+221 117
+221 58
+221 31
+222 362
+222 239
+222 144
+222 124
+222 185
+222 60
+223 136
+223 331
+223 206
+223 367
+223 58
+223 348
+224 65
+224 265
+224 45
+224 77
+224 343
+224 124
+225 260
+225 136
+225 143
+225 276
+225 380
+225 31
+226 294
+226 331
+226 114
+226 243
+226 24
+226 345
+227 200
+227 273
+227 116
+227 55
+227 316
+227 125
+228 176
+228 148
+228 153
+228 348
+228 349
+228 254
+229 326
+229 296
+229 243
+229 276
+229 22
+229 344
+230 161
+230 71
+230 295
+230 361
+230 185
+230 159
+231 130
+231 70
+231 363
+231 79
+231 18
+231 179
+232 121
+232 237
+232 370
+232 83
+232 245
+232 57
+233 355
+233 305
+233 178
+233 308
+233 182
+233 120
+234 263
+234 360
+234 187
+234 114
+234 379
+234 28
+235 110
+235 207
+235 87
+235 312
+235 217
+235 351
+236 160
+236 194
+236 200
+236 78
+236 346
+236 315
+237 67
+237 292
+237 5
+237 232
+237 202
+237 238
+238 261
+238 133
+238 237
+238 80
+238 341
+238 348
+239 195
+239 328
+239 303
+239 176
+239 59
+239 222
+240 328
+240 143
+240 16
+240 177
+240 335
+240 50
+241 133
+241 103
+241 110
+241 177
+241 306
+241 120
+242 8
+242 136
+242 203
+242 215
+242 58
+242 255
+243 226
+243 229
+243 40
+243 136
+243 202
+243 220
+244 129
+244 323
+244 106
+244 362
+244 369
+244 374
+245 193
+245 232
+245 298
+245 11
+245 269
+245 17
+246 161
+246 75
+246 203
+246 251
+246 284
+246 383
+247 4
+247 167
+247 365
+247 368
+247 180
+247 123
+248 70
+248 13
+248 175
+248 211
+248 89
+248 318
+249 291
+249 383
+249 298
+249 339
+249 93
+249 95
+250 64
+250 338
+250 82
+250 189
+250 217
+250 381
+251 256
+251 164
+251 138
+251 307
+251 246
+251 281
+252 39
+252 328
+252 76
+252 205
+252 337
+252 372
+253 96
+253 132
+253 12
+253 76
+253 315
+253 63
+254 193
+254 164
+254 228
+254 36
+254 4
+254 140
+255 327
+255 366
+255 208
+255 242
+255 84
+255 27
+256 2
+256 263
+256 168
+256 51
+256 251
+256 382
+257 128
+257 357
+257 41
+257 275
+257 219
+257 284
+258 288
+258 36
+258 199
+258 362
+258 117
+258 59
+259 193
+259 134
+259 268
+259 364
+259 47
+259 347
+260 225
+260 354
+260 358
+260 266
+260 107
+260 110
+261 44
+261 238
+261 110
+261 144
+261 187
+261 60
+262 320
+262 292
+262 268
+262 207
+262 118
+262 186
+263 256
+263 291
+263 234
+263 371
+263 126
+263 95
+264 131
+264 38
+264 269
+264 152
+264 121
+264 186
+265 224
+265 356
+265 11
+265 48
+265 51
+265 276
+266 32
+266 260
+266 105
+266 138
+266 340
+266 56
+267 132
+267 204
+267 174
+267 368
+267 147
+267 29
+268 3
+268 259
+268 262
+268 39
+268 213
+268 88
+269 192
+269 264
+269 113
+269 148
+269 245
+269 218
+270 194
+270 323
+270 330
+270 301
+270 278
+270 186
+271 320
+271 1
+271 175
+271 180
+271 213
+271 60
+272 1
+272 139
+272 47
+272 303
+272 52
+272 375
+273 227
+273 26
+273 207
+273 340
+273 154
+273 379
+274 357
+274 70
+274 170
+274 364
+274 82
+274 151
+275 161
+275 257
+275 329
+275 368
+275 315
+275 95
+276 225
+276 35
+276 229
+276 265
+276 111
+276 221
+277 128
+277 40
+277 142
+277 79
+277 307
+277 219
+278 295
+278 75
+278 142
+278 270
+278 20
+278 181
+279 100
+279 81
+279 307
+279 150
+279 347
+279 124
+280 64
+280 294
+280 6
+280 200
+280 74
+280 330
+281 39
+281 366
+281 83
+281 61
+281 251
+281 29
+282 1
+282 66
+282 100
+282 79
+282 304
+282 287
+283 11
+283 149
+283 342
+283 54
+283 151
+283 190
+284 257
+284 357
+284 198
+284 368
+284 340
+284 246
+285 329
+285 108
+285 306
+285 179
+285 90
+285 63
+286 70
+286 14
+286 336
+286 89
+286 219
+286 191
+287 128
+287 71
+287 282
+287 378
+287 218
+287 127
+288 258
+288 195
+288 295
+288 7
+288 44
+288 89
+289 165
+289 154
+289 169
+289 364
+289 140
+289 218
+290 33
+290 80
+290 116
+290 149
+290 310
+290 23
+291 324
+291 263
+291 137
+291 249
+291 155
+291 350
+292 262
+292 237
+292 111
+292 176
+292 311
+292 216
+293 196
+293 135
+293 7
+293 297
+293 339
+293 28
+294 226
+294 169
+294 14
+294 47
+294 306
+294 280
+295 288
+295 230
+295 74
+295 278
+295 184
+295 94
+296 99
+296 229
+296 6
+296 333
+296 303
+296 191
+297 131
+297 293
+297 166
+297 13
+297 48
+297 149
+298 99
+298 196
+298 201
+298 245
+298 117
+298 249
+299 102
+299 202
+299 339
+299 371
+299 182
+299 311
+300 356
+300 325
+300 201
+300 301
+300 184
+300 123
+301 9
+301 202
+301 300
+301 142
+301 270
+301 350
+302 129
+302 101
+302 359
+302 200
+302 14
+302 47
+303 296
+303 43
+303 239
+303 272
+303 338
+303 153
+304 1
+304 207
+304 209
+304 282
+304 62
+304 127
+305 65
+305 38
+305 233
+305 18
+305 24
+305 155
+306 294
+306 241
+306 146
+306 371
+306 285
+306 93
+307 109
+307 115
+307 277
+307 279
+307 216
+307 251
+308 34
+308 233
+308 13
+308 366
+308 182
+308 61
+309 159
+309 360
+309 42
+309 15
+309 111
+309 63
+310 290
+310 7
+310 362
+310 332
+310 19
+310 346
+311 292
+311 107
+311 299
+311 86
+311 31
+311 351
+312 363
+312 235
+312 206
+312 155
+312 157
+312 30
+313 2
+313 98
+313 174
+313 116
+313 123
+313 318
+314 35
+314 114
+314 20
+314 150
+314 215
+314 25
+315 40
+315 236
+315 275
+315 52
+315 120
+315 253
+316 353
+316 129
+316 227
+316 321
+316 112
+316 57
+317 116
+317 372
+317 373
+317 344
+317 27
+317 381
+318 326
+318 359
+318 105
+318 248
+318 313
+318 126
+319 33
+319 37
+319 335
+319 16
+319 343
+319 346
+320 352
+320 134
+320 262
+320 173
+320 271
+320 28
+321 194
+321 72
+321 170
+321 75
+321 91
+321 316
+322 78
+322 145
+322 85
+322 23
+322 217
+322 93
+323 68
+323 42
+323 270
+323 338
+323 244
+323 191
+324 97
+324 291
+324 369
+324 18
+324 17
+324 214
+325 300
+325 108
+325 61
+325 181
+325 29
+325 158
+326 229
+326 71
+326 113
+326 150
+326 87
+326 318
+327 130
+327 101
+327 141
+327 341
+327 187
+327 255
+328 160
+328 132
+328 79
+328 240
+328 239
+328 252
+329 166
+329 369
+329 178
+329 275
+329 123
+329 285
+330 99
+330 71
+330 270
+330 373
+330 280
+330 93
+331 226
+331 134
+331 135
+331 169
+331 173
+331 223
+332 3
+332 4
+332 106
+332 172
+332 310
+332 220
+333 35
+333 296
+333 8
+333 360
+333 144
+333 124
+334 64
+334 133
+334 10
+334 144
+334 346
+334 62
+335 205
+335 240
+335 84
+335 182
+335 121
+335 319
+336 354
+336 36
+336 80
+336 347
+336 286
+336 191
+337 164
+337 37
+337 205
+337 83
+337 380
+337 252
+338 323
+338 364
+338 303
+338 145
+338 250
+338 382
+339 293
+339 299
+339 14
+339 114
+339 249
+339 92
+340 266
+340 273
+340 151
+340 26
+340 284
+340 125
+341 327
+341 11
+341 238
+341 20
+341 119
+341 349
+342 354
+342 171
+342 214
+342 120
+342 378
+342 283
+343 32
+343 224
+343 355
+343 80
+343 209
+343 319
+344 35
+344 67
+344 229
+344 82
+344 118
+344 317
+345 226
+345 66
+345 136
+345 139
+345 210
+345 155
+346 236
+346 334
+346 84
+346 310
+346 190
+346 319
+347 259
+347 6
+347 336
+347 85
+347 279
+347 184
+348 192
+348 228
+348 76
+348 238
+348 21
+348 223
+349 228
+349 104
+349 361
+349 372
+349 341
+349 185
+350 291
+350 70
+350 43
+350 301
+350 174
+350 152
+351 72
+351 235
+351 46
+351 143
+351 311
+351 125
+352 320
+352 100
+352 6
+352 39
+352 146
+352 190
+353 356
+353 38
+353 41
+353 141
+353 368
+353 316
+354 2
+354 163
+354 260
+354 336
+354 342
+354 219
+355 233
+355 109
+355 118
+355 343
+355 185
+355 155
+356 353
+356 165
+356 265
+356 300
+356 215
+356 95
+357 257
+357 274
+357 116
+357 378
+357 284
+357 191
+358 260
+358 170
+358 139
+358 14
+358 84
+358 125
+359 360
+359 302
+359 81
+359 376
+359 217
+359 318
+360 359
+360 234
+360 139
+360 333
+360 178
+360 309
+361 230
+361 15
+361 157
+361 374
+361 59
+361 349
+362 258
+362 370
+362 244
+362 310
+362 157
+362 222
+363 193
+363 231
+363 19
+363 312
+363 27
+363 61
+364 289
+364 259
+364 133
+364 111
+364 274
+364 338
+365 102
+365 200
+365 142
+365 53
+365 247
+365 31
+366 121
+366 34
+366 78
+366 308
+366 281
+366 255
+367 97
+367 134
+367 44
+367 184
+367 158
+367 223
+368 353
+368 267
+368 77
+368 275
+368 247
+368 284
+369 324
+369 40
+369 329
+369 144
+369 244
+369 119
+370 161
+370 163
+370 232
+370 362
+370 209
+370 148
+371 98
+371 39
+371 263
+371 299
+371 306
+371 56
+372 41
+372 44
+372 252
+372 125
+372 349
+372 317
+373 330
+373 45
+373 144
+373 375
+373 317
+373 382
+374 5
+374 361
+374 208
+374 244
+374 151
+374 191
+375 132
+375 134
+375 272
+375 177
+375 148
+375 373
+376 5
+376 359
+376 199
+376 75
+376 210
+376 212
+377 73
+377 107
+377 203
+377 173
+377 78
+377 62
+378 357
+378 104
+378 201
+378 342
+378 57
+378 287
+379 68
+379 6
+379 234
+379 209
+379 273
+379 26
+380 225
+380 102
+380 38
+380 15
+380 337
+380 159
+381 32
+381 122
+381 207
+381 250
+381 92
+381 317
+382 256
+382 46
+382 338
+382 83
+382 211
+382 373
+383 135
+383 109
+383 246
+383 24
+383 249
+383 156
diff --git a/random files/96_star.edges b/random files/96_star.edges
new file mode 100644
index 0000000000000000000000000000000000000000..c13fb7be4ce14750d76a8b636a6a666f548e19f6
--- /dev/null
+++ b/random files/96_star.edges
@@ -0,0 +1,191 @@
+96
+0 1
+0 2
+0 3
+0 4
+0 5
+0 6
+0 7
+0 8
+0 9
+0 10
+0 11
+0 12
+0 13
+0 14
+0 15
+0 16
+0 17
+0 18
+0 19
+0 20
+0 21
+0 22
+0 23
+0 24
+0 25
+0 26
+0 27
+0 28
+0 29
+0 30
+0 31
+0 32
+0 33
+0 34
+0 35
+0 36
+0 37
+0 38
+0 39
+0 40
+0 41
+0 42
+0 43
+0 44
+0 45
+0 46
+0 47
+0 48
+0 49
+0 50
+0 51
+0 52
+0 53
+0 54
+0 55
+0 56
+0 57
+0 58
+0 59
+0 60
+0 61
+0 62
+0 63
+0 64
+0 65
+0 66
+0 67
+0 68
+0 69
+0 70
+0 71
+0 72
+0 73
+0 74
+0 75
+0 76
+0 77
+0 78
+0 79
+0 80
+0 81
+0 82
+0 83
+0 84
+0 85
+0 86
+0 87
+0 88
+0 89
+0 90
+0 91
+0 92
+0 93
+0 94
+0 95
+1 0
+2 0
+3 0
+4 0
+5 0
+6 0
+7 0
+8 0
+9 0
+10 0
+11 0
+12 0
+13 0
+14 0
+15 0
+16 0
+17 0
+18 0
+19 0
+20 0
+21 0
+22 0
+23 0
+24 0
+25 0
+26 0
+27 0
+28 0
+29 0
+30 0
+31 0
+32 0
+33 0
+34 0
+35 0
+36 0
+37 0
+38 0
+39 0
+40 0
+41 0
+42 0
+43 0
+44 0
+45 0
+46 0
+47 0
+48 0
+49 0
+50 0
+51 0
+52 0
+53 0
+54 0
+55 0
+56 0
+57 0
+58 0
+59 0
+60 0
+61 0
+62 0
+63 0
+64 0
+65 0
+66 0
+67 0
+68 0
+69 0
+70 0
+71 0
+72 0
+73 0
+74 0
+75 0
+76 0
+77 0
+78 0
+79 0
+80 0
+81 0
+82 0
+83 0
+84 0
+85 0
+86 0
+87 0
+88 0
+89 0
+90 0
+91 0
+92 0
+93 0
+94 0
+95 0
diff --git a/random files/96_star.png b/random files/96_star.png
new file mode 100644
index 0000000000000000000000000000000000000000..09920814dabde2b899fcbe1ab977939ca795db38
Binary files /dev/null and b/random files/96_star.png differ
diff --git a/random files/Diff0:1000.png b/random files/Diff0:1000.png
new file mode 100644
index 0000000000000000000000000000000000000000..b89db60f820b75abeb52eb7692bb84c345d95b8a
Binary files /dev/null and b/random files/Diff0:1000.png differ
diff --git a/random files/FFT_Histogram.png b/random files/FFT_Histogram.png
new file mode 100644
index 0000000000000000000000000000000000000000..87df077759c3759f0a8264d7ceda7ad688dddf41
Binary files /dev/null and b/random files/FFT_Histogram.png differ
diff --git a/random files/Freq_Diff_Values.png b/random files/Freq_Diff_Values.png
new file mode 100644
index 0000000000000000000000000000000000000000..0019df17b23b220780edddd4fd9a693cc1934284
Binary files /dev/null and b/random files/Freq_Diff_Values.png differ
diff --git a/random files/Freq_Values.png b/random files/Freq_Values.png
new file mode 100644
index 0000000000000000000000000000000000000000..0530030d053b6f04262a90115ecb68303d9cf36f
Binary files /dev/null and b/random files/Freq_Values.png differ
diff --git a/random files/Gradient_Values.png b/random files/Gradient_Values.png
new file mode 100644
index 0000000000000000000000000000000000000000..a16236956d202b260d4dccb3ec53d761a8072d0f
Binary files /dev/null and b/random files/Gradient_Values.png differ
diff --git a/random files/Hist.png b/random files/Hist.png
new file mode 100644
index 0000000000000000000000000000000000000000..80505aed9b005f27275711e8fdc1db29bdf8e287
Binary files /dev/null and b/random files/Hist.png differ
diff --git a/random files/Parameter_Frequency.png b/random files/Parameter_Frequency.png
new file mode 100644
index 0000000000000000000000000000000000000000..d32539788254089c650aa9666fcc4888d2186c27
Binary files /dev/null and b/random files/Parameter_Frequency.png differ
diff --git a/random files/Parameter_Histogram.png b/random files/Parameter_Histogram.png
new file mode 100644
index 0000000000000000000000000000000000000000..1a8e0944b86812ba245c508434b8bf47e5a9a2e0
Binary files /dev/null and b/random files/Parameter_Histogram.png differ
diff --git a/random files/Parameter_Values.png b/random files/Parameter_Values.png
new file mode 100644
index 0000000000000000000000000000000000000000..395ff86c53ab932e2d508d214f5d4ebe8ffdbcef
Binary files /dev/null and b/random files/Parameter_Values.png differ
diff --git a/random files/Parameters.png b/random files/Parameters.png
new file mode 100644
index 0000000000000000000000000000000000000000..6e062d0496d180109da09ff68cee1bc248010e69
Binary files /dev/null and b/random files/Parameters.png differ
diff --git a/random files/ParametersWaveletHaar.png b/random files/ParametersWaveletHaar.png
new file mode 100644
index 0000000000000000000000000000000000000000..b89db60f820b75abeb52eb7692bb84c345d95b8a
Binary files /dev/null and b/random files/ParametersWaveletHaar.png differ
diff --git a/random files/ParametersWaveletHaar.svg b/random files/ParametersWaveletHaar.svg
new file mode 100644
index 0000000000000000000000000000000000000000..f65da72e8eafd16892bad85a57bcf03b5ea92b86
--- /dev/null
+++ b/random files/ParametersWaveletHaar.svg
@@ -0,0 +1,2332 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="432pt" height="288pt" viewBox="0 0 432 288" xmlns="http://www.w3.org/2000/svg" version="1.1">
+ <metadata>
+ <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+ <cc:Work>
+ <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
+ <dc:date>2022-03-23T20:37:51.057299</dc:date>
+ <dc:format>image/svg+xml</dc:format>
+ <dc:creator>
+ <cc:Agent>
+ <dc:title>Matplotlib v3.5.1, https://matplotlib.org/</dc:title>
+ </cc:Agent>
+ </dc:creator>
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <defs>
+ <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style>
+ </defs>
+ <g id="figure_1">
+ <g id="patch_1">
+ <path d="M 0 288
+L 432 288
+L 432 0
+L 0 0
+L 0 288
+z
+" style="fill: none"/>
+ </g>
+ <g id="axes_1">
+ <g id="patch_2">
+ <path d="M 54 252
+L 388.8 252
+L 388.8 34.56
+L 54 34.56
+z
+" style="fill: #ffffff"/>
+ </g>
+ <g id="matplotlib.axis_1">
+ <g id="xtick_1">
+ <g id="line2d_1">
+ <defs>
+ <path id="m20d77cf94f" d="M 0 0
+L 0 3.5
+" style="stroke: #000000; stroke-width: 0.8"/>
+ </defs>
+ <g>
+ <use xlink:href="#m20d77cf94f" x="69.218182" y="252" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_1">
+ <!-- 40000 -->
+ <g transform="translate(53.311932 266.598437)scale(0.1 -0.1)">
+ <defs>
+ <path id="DejaVuSans-34" d="M 2419 4116
+L 825 1625
+L 2419 1625
+L 2419 4116
+z
+M 2253 4666
+L 3047 4666
+L 3047 1625
+L 3713 1625
+L 3713 1100
+L 3047 1100
+L 3047 0
+L 2419 0
+L 2419 1100
+L 313 1100
+L 313 1709
+L 2253 4666
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-30" d="M 2034 4250
+Q 1547 4250 1301 3770
+Q 1056 3291 1056 2328
+Q 1056 1369 1301 889
+Q 1547 409 2034 409
+Q 2525 409 2770 889
+Q 3016 1369 3016 2328
+Q 3016 3291 2770 3770
+Q 2525 4250 2034 4250
+z
+M 2034 4750
+Q 2819 4750 3233 4129
+Q 3647 3509 3647 2328
+Q 3647 1150 3233 529
+Q 2819 -91 2034 -91
+Q 1250 -91 836 529
+Q 422 1150 422 2328
+Q 422 3509 836 4129
+Q 1250 4750 2034 4750
+z
+" transform="scale(0.015625)"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-34"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-30" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ <use xlink:href="#DejaVuSans-30" x="254.492188"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_2">
+ <g id="line2d_2">
+ <g>
+ <use xlink:href="#m20d77cf94f" x="130.151843" y="252" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_2">
+ <!-- 40200 -->
+ <g transform="translate(114.245593 266.598437)scale(0.1 -0.1)">
+ <defs>
+ <path id="DejaVuSans-32" d="M 1228 531
+L 3431 531
+L 3431 0
+L 469 0
+L 469 531
+Q 828 903 1448 1529
+Q 2069 2156 2228 2338
+Q 2531 2678 2651 2914
+Q 2772 3150 2772 3378
+Q 2772 3750 2511 3984
+Q 2250 4219 1831 4219
+Q 1534 4219 1204 4116
+Q 875 4013 500 3803
+L 500 4441
+Q 881 4594 1212 4672
+Q 1544 4750 1819 4750
+Q 2544 4750 2975 4387
+Q 3406 4025 3406 3419
+Q 3406 3131 3298 2873
+Q 3191 2616 2906 2266
+Q 2828 2175 2409 1742
+Q 1991 1309 1228 531
+z
+" transform="scale(0.015625)"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-34"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-32" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ <use xlink:href="#DejaVuSans-30" x="254.492188"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_3">
+ <g id="line2d_3">
+ <g>
+ <use xlink:href="#m20d77cf94f" x="191.085504" y="252" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_3">
+ <!-- 40400 -->
+ <g transform="translate(175.179254 266.598437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-34"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-34" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ <use xlink:href="#DejaVuSans-30" x="254.492188"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_4">
+ <g id="line2d_4">
+ <g>
+ <use xlink:href="#m20d77cf94f" x="252.019165" y="252" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_4">
+ <!-- 40600 -->
+ <g transform="translate(236.112915 266.598437)scale(0.1 -0.1)">
+ <defs>
+ <path id="DejaVuSans-36" d="M 2113 2584
+Q 1688 2584 1439 2293
+Q 1191 2003 1191 1497
+Q 1191 994 1439 701
+Q 1688 409 2113 409
+Q 2538 409 2786 701
+Q 3034 994 3034 1497
+Q 3034 2003 2786 2293
+Q 2538 2584 2113 2584
+z
+M 3366 4563
+L 3366 3988
+Q 3128 4100 2886 4159
+Q 2644 4219 2406 4219
+Q 1781 4219 1451 3797
+Q 1122 3375 1075 2522
+Q 1259 2794 1537 2939
+Q 1816 3084 2150 3084
+Q 2853 3084 3261 2657
+Q 3669 2231 3669 1497
+Q 3669 778 3244 343
+Q 2819 -91 2113 -91
+Q 1303 -91 875 529
+Q 447 1150 447 2328
+Q 447 3434 972 4092
+Q 1497 4750 2381 4750
+Q 2619 4750 2861 4703
+Q 3103 4656 3366 4563
+z
+" transform="scale(0.015625)"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-34"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-36" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ <use xlink:href="#DejaVuSans-30" x="254.492188"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_5">
+ <g id="line2d_5">
+ <g>
+ <use xlink:href="#m20d77cf94f" x="312.952826" y="252" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_5">
+ <!-- 40800 -->
+ <g transform="translate(297.046576 266.598437)scale(0.1 -0.1)">
+ <defs>
+ <path id="DejaVuSans-38" d="M 2034 2216
+Q 1584 2216 1326 1975
+Q 1069 1734 1069 1313
+Q 1069 891 1326 650
+Q 1584 409 2034 409
+Q 2484 409 2743 651
+Q 3003 894 3003 1313
+Q 3003 1734 2745 1975
+Q 2488 2216 2034 2216
+z
+M 1403 2484
+Q 997 2584 770 2862
+Q 544 3141 544 3541
+Q 544 4100 942 4425
+Q 1341 4750 2034 4750
+Q 2731 4750 3128 4425
+Q 3525 4100 3525 3541
+Q 3525 3141 3298 2862
+Q 3072 2584 2669 2484
+Q 3125 2378 3379 2068
+Q 3634 1759 3634 1313
+Q 3634 634 3220 271
+Q 2806 -91 2034 -91
+Q 1263 -91 848 271
+Q 434 634 434 1313
+Q 434 1759 690 2068
+Q 947 2378 1403 2484
+z
+M 1172 3481
+Q 1172 3119 1398 2916
+Q 1625 2713 2034 2713
+Q 2441 2713 2670 2916
+Q 2900 3119 2900 3481
+Q 2900 3844 2670 4047
+Q 2441 4250 2034 4250
+Q 1625 4250 1398 4047
+Q 1172 3844 1172 3481
+z
+" transform="scale(0.015625)"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-34"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-38" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ <use xlink:href="#DejaVuSans-30" x="254.492188"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_6">
+ <g id="line2d_6">
+ <g>
+ <use xlink:href="#m20d77cf94f" x="373.886486" y="252" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_6">
+ <!-- 41000 -->
+ <g transform="translate(357.980236 266.598437)scale(0.1 -0.1)">
+ <defs>
+ <path id="DejaVuSans-31" d="M 794 531
+L 1825 531
+L 1825 4091
+L 703 3866
+L 703 4441
+L 1819 4666
+L 2450 4666
+L 2450 531
+L 3481 531
+L 3481 0
+L 794 0
+L 794 531
+z
+" transform="scale(0.015625)"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-34"/>
+ <use xlink:href="#DejaVuSans-31" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-30" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ <use xlink:href="#DejaVuSans-30" x="254.492188"/>
+ </g>
+ </g>
+ </g>
+ <g id="text_7">
+ <!-- Parameter indices -->
+ <g transform="translate(176.246875 280.276563)scale(0.1 -0.1)">
+ <defs>
+ <path id="DejaVuSans-50" d="M 1259 4147
+L 1259 2394
+L 2053 2394
+Q 2494 2394 2734 2622
+Q 2975 2850 2975 3272
+Q 2975 3691 2734 3919
+Q 2494 4147 2053 4147
+L 1259 4147
+z
+M 628 4666
+L 2053 4666
+Q 2838 4666 3239 4311
+Q 3641 3956 3641 3272
+Q 3641 2581 3239 2228
+Q 2838 1875 2053 1875
+L 1259 1875
+L 1259 0
+L 628 0
+L 628 4666
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-61" d="M 2194 1759
+Q 1497 1759 1228 1600
+Q 959 1441 959 1056
+Q 959 750 1161 570
+Q 1363 391 1709 391
+Q 2188 391 2477 730
+Q 2766 1069 2766 1631
+L 2766 1759
+L 2194 1759
+z
+M 3341 1997
+L 3341 0
+L 2766 0
+L 2766 531
+Q 2569 213 2275 61
+Q 1981 -91 1556 -91
+Q 1019 -91 701 211
+Q 384 513 384 1019
+Q 384 1609 779 1909
+Q 1175 2209 1959 2209
+L 2766 2209
+L 2766 2266
+Q 2766 2663 2505 2880
+Q 2244 3097 1772 3097
+Q 1472 3097 1187 3025
+Q 903 2953 641 2809
+L 641 3341
+Q 956 3463 1253 3523
+Q 1550 3584 1831 3584
+Q 2591 3584 2966 3190
+Q 3341 2797 3341 1997
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-72" d="M 2631 2963
+Q 2534 3019 2420 3045
+Q 2306 3072 2169 3072
+Q 1681 3072 1420 2755
+Q 1159 2438 1159 1844
+L 1159 0
+L 581 0
+L 581 3500
+L 1159 3500
+L 1159 2956
+Q 1341 3275 1631 3429
+Q 1922 3584 2338 3584
+Q 2397 3584 2469 3576
+Q 2541 3569 2628 3553
+L 2631 2963
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-6d" d="M 3328 2828
+Q 3544 3216 3844 3400
+Q 4144 3584 4550 3584
+Q 5097 3584 5394 3201
+Q 5691 2819 5691 2113
+L 5691 0
+L 5113 0
+L 5113 2094
+Q 5113 2597 4934 2840
+Q 4756 3084 4391 3084
+Q 3944 3084 3684 2787
+Q 3425 2491 3425 1978
+L 3425 0
+L 2847 0
+L 2847 2094
+Q 2847 2600 2669 2842
+Q 2491 3084 2119 3084
+Q 1678 3084 1418 2786
+Q 1159 2488 1159 1978
+L 1159 0
+L 581 0
+L 581 3500
+L 1159 3500
+L 1159 2956
+Q 1356 3278 1631 3431
+Q 1906 3584 2284 3584
+Q 2666 3584 2933 3390
+Q 3200 3197 3328 2828
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-65" d="M 3597 1894
+L 3597 1613
+L 953 1613
+Q 991 1019 1311 708
+Q 1631 397 2203 397
+Q 2534 397 2845 478
+Q 3156 559 3463 722
+L 3463 178
+Q 3153 47 2828 -22
+Q 2503 -91 2169 -91
+Q 1331 -91 842 396
+Q 353 884 353 1716
+Q 353 2575 817 3079
+Q 1281 3584 2069 3584
+Q 2775 3584 3186 3129
+Q 3597 2675 3597 1894
+z
+M 3022 2063
+Q 3016 2534 2758 2815
+Q 2500 3097 2075 3097
+Q 1594 3097 1305 2825
+Q 1016 2553 972 2059
+L 3022 2063
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-74" d="M 1172 4494
+L 1172 3500
+L 2356 3500
+L 2356 3053
+L 1172 3053
+L 1172 1153
+Q 1172 725 1289 603
+Q 1406 481 1766 481
+L 2356 481
+L 2356 0
+L 1766 0
+Q 1100 0 847 248
+Q 594 497 594 1153
+L 594 3053
+L 172 3053
+L 172 3500
+L 594 3500
+L 594 4494
+L 1172 4494
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-20" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-69" d="M 603 3500
+L 1178 3500
+L 1178 0
+L 603 0
+L 603 3500
+z
+M 603 4863
+L 1178 4863
+L 1178 4134
+L 603 4134
+L 603 4863
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-6e" d="M 3513 2113
+L 3513 0
+L 2938 0
+L 2938 2094
+Q 2938 2591 2744 2837
+Q 2550 3084 2163 3084
+Q 1697 3084 1428 2787
+Q 1159 2491 1159 1978
+L 1159 0
+L 581 0
+L 581 3500
+L 1159 3500
+L 1159 2956
+Q 1366 3272 1645 3428
+Q 1925 3584 2291 3584
+Q 2894 3584 3203 3211
+Q 3513 2838 3513 2113
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-64" d="M 2906 2969
+L 2906 4863
+L 3481 4863
+L 3481 0
+L 2906 0
+L 2906 525
+Q 2725 213 2448 61
+Q 2172 -91 1784 -91
+Q 1150 -91 751 415
+Q 353 922 353 1747
+Q 353 2572 751 3078
+Q 1150 3584 1784 3584
+Q 2172 3584 2448 3432
+Q 2725 3281 2906 2969
+z
+M 947 1747
+Q 947 1113 1208 752
+Q 1469 391 1925 391
+Q 2381 391 2643 752
+Q 2906 1113 2906 1747
+Q 2906 2381 2643 2742
+Q 2381 3103 1925 3103
+Q 1469 3103 1208 2742
+Q 947 2381 947 1747
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-63" d="M 3122 3366
+L 3122 2828
+Q 2878 2963 2633 3030
+Q 2388 3097 2138 3097
+Q 1578 3097 1268 2742
+Q 959 2388 959 1747
+Q 959 1106 1268 751
+Q 1578 397 2138 397
+Q 2388 397 2633 464
+Q 2878 531 3122 666
+L 3122 134
+Q 2881 22 2623 -34
+Q 2366 -91 2075 -91
+Q 1284 -91 818 406
+Q 353 903 353 1747
+Q 353 2603 823 3093
+Q 1294 3584 2113 3584
+Q 2378 3584 2631 3529
+Q 2884 3475 3122 3366
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-73" d="M 2834 3397
+L 2834 2853
+Q 2591 2978 2328 3040
+Q 2066 3103 1784 3103
+Q 1356 3103 1142 2972
+Q 928 2841 928 2578
+Q 928 2378 1081 2264
+Q 1234 2150 1697 2047
+L 1894 2003
+Q 2506 1872 2764 1633
+Q 3022 1394 3022 966
+Q 3022 478 2636 193
+Q 2250 -91 1575 -91
+Q 1294 -91 989 -36
+Q 684 19 347 128
+L 347 722
+Q 666 556 975 473
+Q 1284 391 1588 391
+Q 1994 391 2212 530
+Q 2431 669 2431 922
+Q 2431 1156 2273 1281
+Q 2116 1406 1581 1522
+L 1381 1569
+Q 847 1681 609 1914
+Q 372 2147 372 2553
+Q 372 3047 722 3315
+Q 1072 3584 1716 3584
+Q 2034 3584 2315 3537
+Q 2597 3491 2834 3397
+z
+" transform="scale(0.015625)"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-50"/>
+ <use xlink:href="#DejaVuSans-61" x="55.802734"/>
+ <use xlink:href="#DejaVuSans-72" x="117.082031"/>
+ <use xlink:href="#DejaVuSans-61" x="158.195312"/>
+ <use xlink:href="#DejaVuSans-6d" x="219.474609"/>
+ <use xlink:href="#DejaVuSans-65" x="316.886719"/>
+ <use xlink:href="#DejaVuSans-74" x="378.410156"/>
+ <use xlink:href="#DejaVuSans-65" x="417.619141"/>
+ <use xlink:href="#DejaVuSans-72" x="479.142578"/>
+ <use xlink:href="#DejaVuSans-20" x="520.255859"/>
+ <use xlink:href="#DejaVuSans-69" x="552.042969"/>
+ <use xlink:href="#DejaVuSans-6e" x="579.826172"/>
+ <use xlink:href="#DejaVuSans-64" x="643.205078"/>
+ <use xlink:href="#DejaVuSans-69" x="706.681641"/>
+ <use xlink:href="#DejaVuSans-63" x="734.464844"/>
+ <use xlink:href="#DejaVuSans-65" x="789.445312"/>
+ <use xlink:href="#DejaVuSans-73" x="850.96875"/>
+ </g>
+ </g>
+ </g>
+ <g id="matplotlib.axis_2">
+ <g id="ytick_1">
+ <g id="line2d_7">
+ <defs>
+ <path id="mcae92e3e51" d="M 0 0
+L -3.5 0
+" style="stroke: #000000; stroke-width: 0.8"/>
+ </defs>
+ <g>
+ <use xlink:href="#mcae92e3e51" x="54" y="242.910438" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_8">
+ <!-- 0 -->
+ <g transform="translate(40.6375 246.709657)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-30"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_2">
+ <g id="line2d_8">
+ <g>
+ <use xlink:href="#mcae92e3e51" x="54" y="208.385466" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_9">
+ <!-- 2000 -->
+ <g transform="translate(21.55 212.184685)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-32"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-30" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_3">
+ <g id="line2d_9">
+ <g>
+ <use xlink:href="#mcae92e3e51" x="54" y="173.860494" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_10">
+ <!-- 4000 -->
+ <g transform="translate(21.55 177.659713)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-34"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-30" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_4">
+ <g id="line2d_10">
+ <g>
+ <use xlink:href="#mcae92e3e51" x="54" y="139.335522" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_11">
+ <!-- 6000 -->
+ <g transform="translate(21.55 143.134741)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-36"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-30" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_5">
+ <g id="line2d_11">
+ <g>
+ <use xlink:href="#mcae92e3e51" x="54" y="104.81055" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_12">
+ <!-- 8000 -->
+ <g transform="translate(21.55 108.609769)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-38"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-30" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_6">
+ <g id="line2d_12">
+ <g>
+ <use xlink:href="#mcae92e3e51" x="54" y="70.285578" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_13">
+ <!-- 10000 -->
+ <g transform="translate(15.1875 74.084797)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-31"/>
+ <use xlink:href="#DejaVuSans-30" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-30" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ <use xlink:href="#DejaVuSans-30" x="254.492188"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_7">
+ <g id="line2d_13">
+ <g>
+ <use xlink:href="#mcae92e3e51" x="54" y="35.760606" style="stroke: #000000; stroke-width: 0.8"/>
+ </g>
+ </g>
+ <g id="text_14">
+ <!-- 12000 -->
+ <g transform="translate(15.1875 39.559825)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-31"/>
+ <use xlink:href="#DejaVuSans-32" x="63.623047"/>
+ <use xlink:href="#DejaVuSans-30" x="127.246094"/>
+ <use xlink:href="#DejaVuSans-30" x="190.869141"/>
+ <use xlink:href="#DejaVuSans-30" x="254.492188"/>
+ </g>
+ </g>
+ </g>
+ </g>
+ <g id="line2d_14">
+ <path d="M 69.218182 241.719326
+L 69.827518 241.788376
+L 70.132187 241.719326
+L 71.046192 241.805639
+L 71.35086 241.408602
+L 71.655528 241.477652
+L 71.960197 241.788376
+L 72.569533 241.702064
+L 72.874201 241.391339
+L 73.17887 241.374077
+L 73.483538 241.719326
+L 74.397543 241.736589
+L 75.00688 241.512177
+L 75.920885 241.719326
+L 76.225553 241.546702
+L 77.444226 241.719326
+L 77.748894 241.529439
+L 78.358231 241.563964
+L 78.662899 241.581227
+L 79.272236 241.218714
+L 79.881572 241.391339
+L 80.186241 241.339552
+L 80.490909 241.667539
+L 80.795577 241.598489
+L 81.404914 241.650277
+L 81.709582 241.667539
+L 82.014251 241.995526
+L 82.318919 241.943739
+L 82.928256 241.995526
+L 83.232924 240.76989
+L 83.537592 242.030051
+L 84.146929 241.926476
+L 84.451597 241.719326
+L 84.756265 241.028827
+L 85.060934 242.012789
+L 85.365602 241.270502
+L 85.67027 241.477652
+L 85.974939 240.70084
+L 86.279607 241.166927
+L 86.584275 241.391339
+L 87.193612 241.374077
+L 87.49828 241.305027
+L 87.802948 241.425864
+L 88.107617 241.736589
+L 88.716953 241.736589
+L 89.021622 241.719326
+L 89.32629 240.528215
+L 89.630958 241.961001
+L 90.240295 241.719326
+L 90.544963 241.04609
+L 90.849631 240.683577
+L 91.1543 241.822901
+L 91.458968 241.961001
+L 91.763636 241.028827
+L 92.068305 241.115139
+L 92.372973 240.735365
+L 92.677641 241.857426
+L 92.98231 241.961001
+L 93.591646 240.476427
+L 93.896314 240.718102
+L 94.200983 242.047314
+L 96.638329 241.995526
+L 96.942998 241.080614
+L 97.247666 241.667539
+L 97.552334 241.460389
+L 97.857002 241.477652
+L 98.161671 240.56274
+L 98.466339 240.666315
+L 98.771007 242.047314
+L 99.075676 241.529439
+L 99.380344 240.528215
+L 99.989681 240.56274
+L 100.294349 242.099101
+L 100.903686 240.49369
+L 101.208354 240.787152
+L 101.513022 240.580002
+L 101.81769 241.218714
+L 102.122359 241.115139
+L 102.427027 241.253239
+L 103.036364 241.184189
+L 103.341032 241.529439
+L 104.559705 241.529439
+L 104.864373 241.04609
+L 105.169042 240.856202
+L 105.778378 240.90799
+L 106.083047 240.718102
+L 106.387715 241.563964
+L 106.997052 241.512177
+L 107.606388 241.063352
+L 107.911057 241.529439
+L 108.215725 241.149664
+L 108.520393 241.305027
+L 108.825061 241.201452
+L 109.12973 241.305027
+L 109.434398 240.890727
+L 109.739066 241.028827
+L 110.348403 240.994302
+L 110.653071 240.890727
+L 110.95774 241.166927
+L 111.262408 241.253239
+L 111.871744 241.149664
+L 112.176413 241.166927
+L 112.481081 241.529439
+L 112.785749 241.408602
+L 113.090418 241.494914
+L 113.395086 241.443127
+L 113.699754 241.563964
+L 114.004423 241.460389
+L 114.309091 241.512177
+L 114.613759 241.149664
+L 115.223096 241.253239
+L 115.527764 241.494914
+L 116.137101 240.597265
+L 116.441769 240.597265
+L 116.746437 240.390115
+L 117.051106 241.339552
+L 117.965111 241.235977
+L 118.269779 241.391339
+L 118.574447 241.874689
+L 118.879115 241.961001
+L 119.183784 241.891951
+L 119.488452 242.030051
+L 119.79312 240.683577
+L 120.097789 241.477652
+L 120.707125 241.581227
+L 121.316462 240.21749
+L 121.62113 242.099101
+L 121.925799 242.099101
+L 122.230467 240.873465
+L 122.535135 240.70084
+L 122.839803 240.942515
+L 123.144472 241.943739
+L 123.44914 241.667539
+L 123.753808 240.90799
+L 124.058477 240.614527
+L 124.363145 240.56274
+L 124.667813 241.080614
+L 125.27715 240.90799
+L 125.886486 240.90799
+L 126.191155 241.253239
+L 126.800491 241.287764
+L 127.10516 240.942515
+L 127.409828 240.234753
+L 127.714496 241.494914
+L 128.323833 240.821677
+L 128.628501 240.63179
+L 128.93317 241.115139
+L 129.542506 240.890727
+L 129.847174 240.390115
+L 130.151843 240.597265
+L 130.456511 240.597265
+L 130.761179 240.856202
+L 131.370516 240.131178
+L 131.675184 240.787152
+L 131.979853 240.76989
+L 132.284521 240.165703
+L 132.589189 240.269278
+L 132.893857 240.165703
+L 134.112531 240.303803
+L 134.417199 240.165703
+L 134.721867 240.390115
+L 135.026536 240.14844
+L 135.331204 240.321065
+L 135.940541 240.35559
+L 136.549877 240.42464
+L 136.854545 240.649052
+L 137.463882 240.545477
+L 137.76855 241.063352
+L 138.073219 240.83894
+L 138.377887 240.441902
+L 138.987224 240.735365
+L 139.291892 240.752627
+L 139.901229 241.115139
+L 140.815233 241.04609
+L 141.119902 240.925252
+L 141.42457 241.080614
+L 142.338575 241.028827
+L 142.643243 240.49369
+L 142.947912 240.994302
+L 143.557248 240.994302
+L 143.861916 240.459165
+L 144.166585 240.28654
+L 144.471253 241.356814
+L 144.775921 240.70084
+L 145.385258 240.21749
+L 145.689926 240.76989
+L 145.994595 240.76989
+L 146.299263 240.649052
+L 146.603931 240.252015
+L 146.9086 240.407378
+L 147.213268 240.131178
+L 147.517936 241.132402
+L 148.127273 241.080614
+L 148.736609 241.115139
+L 149.041278 241.684801
+L 151.173956 241.374077
+L 151.478624 240.890727
+L 151.783292 240.890727
+L 152.087961 241.788376
+L 152.392629 241.753851
+L 152.697297 241.494914
+L 153.001966 240.303803
+L 153.306634 240.338328
+L 153.611302 241.667539
+L 153.915971 240.752627
+L 154.220639 240.372853
+L 154.525307 240.994302
+L 154.829975 240.942515
+L 155.134644 241.322289
+L 157.267322 241.287764
+L 157.57199 240.994302
+L 157.876658 241.028827
+L 158.181327 241.633014
+L 158.485995 241.598489
+L 159.095332 240.959777
+L 159.4 241.650277
+L 160.009337 241.460389
+L 160.314005 240.735365
+L 160.923342 240.666315
+L 161.22801 241.132402
+L 161.532678 240.441902
+L 162.446683 240.390115
+L 162.751351 241.011565
+L 163.970025 240.97704
+L 164.884029 240.994302
+L 165.188698 240.83894
+L 165.493366 240.942515
+L 165.798034 241.339552
+L 166.712039 241.287764
+L 167.016708 241.581227
+L 167.626044 241.080614
+L 167.930713 241.356814
+L 168.235381 241.356814
+L 168.844717 241.218714
+L 169.149386 241.270502
+L 169.454054 241.080614
+L 170.063391 240.97704
+L 170.368059 241.615752
+L 172.805405 241.650277
+L 174.024079 241.408602
+L 174.633415 240.372853
+L 174.938084 241.874689
+L 175.54742 241.201452
+L 175.852088 240.131178
+L 176.156757 240.321065
+L 176.461425 241.253239
+L 177.070762 240.407378
+L 177.680098 240.407378
+L 177.984767 240.90799
+L 178.594103 240.856202
+L 178.898771 240.580002
+L 179.20344 240.925252
+L 179.508108 240.925252
+L 179.812776 241.04609
+L 180.422113 240.959777
+L 180.726781 241.253239
+L 181.640786 240.97704
+L 181.945455 240.718102
+L 182.250123 240.735365
+L 182.554791 241.305027
+L 182.859459 241.235977
+L 183.468796 240.459165
+L 183.773464 241.218714
+L 184.078133 241.235977
+L 184.687469 240.252015
+L 185.296806 240.890727
+L 185.601474 240.528215
+L 187.124816 240.597265
+L 187.734152 240.63179
+L 188.038821 240.510952
+L 188.648157 240.76989
+L 188.952826 240.735365
+L 189.257494 240.873465
+L 189.562162 240.649052
+L 189.86683 239.87224
+L 190.171499 240.804415
+L 190.476167 240.942515
+L 190.780835 240.83894
+L 191.390172 240.269278
+L 191.69484 240.890727
+L 191.999509 240.873465
+L 192.304177 240.321065
+L 192.608845 240.234753
+L 192.913514 240.269278
+L 193.218182 241.149664
+L 193.52285 240.959777
+L 193.827518 241.04609
+L 194.132187 240.90799
+L 194.436855 240.942515
+L 194.741523 241.201452
+L 195.655528 241.149664
+L 195.960197 240.787152
+L 196.874201 240.90799
+L 197.17887 240.666315
+L 197.483538 240.027603
+L 197.788206 241.305027
+L 198.092875 241.115139
+L 198.702211 240.476427
+L 199.00688 240.49369
+L 199.311548 241.063352
+L 199.616216 241.080614
+L 199.920885 241.581227
+L 200.225553 240.372853
+L 200.530221 240.269278
+L 200.834889 240.959777
+L 201.139558 241.080614
+L 201.748894 241.063352
+L 202.053563 241.115139
+L 202.358231 240.83894
+L 202.662899 241.132402
+L 202.967568 240.76989
+L 203.272236 241.080614
+L 203.576904 241.011565
+L 203.881572 241.097877
+L 204.490909 240.821677
+L 204.795577 240.269278
+L 205.100246 240.14844
+L 205.404914 241.425864
+L 205.709582 241.374077
+L 206.014251 240.476427
+L 206.623587 240.787152
+L 206.928256 241.356814
+L 207.232924 240.76989
+L 207.537592 239.889503
+L 208.146929 240.113915
+L 208.451597 241.097877
+L 208.756265 241.253239
+L 209.67027 241.115139
+L 209.974939 241.719326
+L 210.584275 241.771114
+L 210.888943 241.494914
+L 211.193612 241.771114
+L 212.107617 241.788376
+L 212.412285 240.890727
+L 212.716953 240.372853
+L 213.021622 241.771114
+L 213.32629 240.873465
+L 213.630958 240.90799
+L 213.935627 240.597265
+L 214.240295 240.545477
+L 214.544963 241.443127
+L 214.849631 240.63179
+L 215.1543 240.252015
+L 215.763636 240.044865
+L 216.068305 241.149664
+L 216.372973 241.080614
+L 216.677641 240.804415
+L 216.98231 241.132402
+L 217.286978 241.132402
+L 217.591646 241.391339
+L 218.200983 241.391339
+L 218.505651 241.097877
+L 218.810319 240.338328
+L 219.114988 241.771114
+L 219.419656 241.719326
+L 219.724324 241.374077
+L 220.028993 240.476427
+L 220.333661 240.42464
+L 220.638329 241.771114
+L 221.247666 240.027603
+L 221.552334 240.545477
+L 221.857002 240.459165
+L 222.161671 241.477652
+L 222.466339 240.113915
+L 222.771007 239.94129
+L 223.075676 240.234753
+L 223.380344 240.269278
+L 223.685012 240.925252
+L 224.903686 240.856202
+L 225.208354 241.253239
+L 225.513022 241.201452
+L 226.122359 241.322289
+L 226.427027 241.287764
+L 226.731695 241.615752
+L 227.036364 241.546702
+L 227.950369 241.719326
+L 228.864373 241.529439
+L 229.169042 241.546702
+L 229.47371 241.235977
+L 229.778378 241.512177
+L 230.083047 240.994302
+L 230.997052 240.321065
+L 231.30172 241.201452
+L 232.520393 241.132402
+L 232.825061 241.477652
+L 233.12973 241.512177
+L 233.434398 241.425864
+L 233.739066 241.563964
+L 234.043735 241.546702
+L 234.348403 241.736589
+L 234.653071 241.598489
+L 235.262408 241.581227
+L 235.567076 240.76989
+L 235.871744 241.857426
+L 236.176413 241.822901
+L 236.481081 241.322289
+L 236.785749 241.235977
+L 237.090418 240.994302
+L 237.395086 241.719326
+L 238.004423 241.149664
+L 238.309091 240.372853
+L 238.613759 240.113915
+L 238.918428 240.649052
+L 239.223096 240.70084
+L 239.527764 240.614527
+L 239.832432 240.683577
+L 240.441769 240.56274
+L 240.746437 240.735365
+L 241.051106 240.666315
+L 241.660442 240.252015
+L 241.965111 240.614527
+L 242.574447 240.338328
+L 242.879115 240.459165
+L 243.183784 240.28654
+L 243.488452 240.683577
+L 243.79312 240.735365
+L 244.097789 240.269278
+L 244.402457 240.338328
+L 244.707125 240.14844
+L 245.316462 240.994302
+L 245.62113 240.873465
+L 245.925799 240.303803
+L 246.230467 240.165703
+L 246.535135 240.407378
+L 247.753808 240.372853
+L 248.058477 240.735365
+L 249.581818 240.683577
+L 250.191155 240.49369
+L 250.495823 240.597265
+L 250.800491 240.165703
+L 251.10516 241.080614
+L 251.409828 240.959777
+L 251.714496 240.580002
+L 252.019165 240.614527
+L 252.323833 240.407378
+L 252.628501 241.04609
+L 252.93317 240.70084
+L 253.237838 240.044865
+L 253.542506 240.459165
+L 253.847174 240.597265
+L 254.151843 241.391339
+L 254.456511 241.581227
+L 255.065848 241.408602
+L 255.370516 241.425864
+L 255.675184 241.598489
+L 257.807862 241.615752
+L 258.112531 241.425864
+L 258.417199 241.097877
+L 258.721867 241.719326
+L 259.026536 241.460389
+L 259.331204 241.529439
+L 259.635872 240.56274
+L 259.940541 240.614527
+L 260.245209 241.633014
+L 261.159214 240.90799
+L 261.463882 240.683577
+L 261.76855 241.080614
+L 262.073219 240.994302
+L 262.377887 241.080614
+L 262.682555 241.011565
+L 262.987224 241.184189
+L 263.291892 241.063352
+L 263.901229 241.115139
+L 264.205897 240.752627
+L 264.510565 240.890727
+L 264.815233 241.339552
+L 265.42457 241.253239
+L 266.033907 240.07939
+L 266.338575 241.512177
+L 266.643243 240.873465
+L 266.947912 241.063352
+L 267.25258 240.63179
+L 267.557248 240.459165
+L 267.861916 241.322289
+L 268.166585 240.76989
+L 268.471253 239.958553
+L 268.775921 240.200228
+L 269.08059 240.200228
+L 269.385258 241.650277
+L 271.213268 241.633014
+L 271.822604 241.598489
+L 272.127273 240.252015
+L 272.431941 241.356814
+L 273.041278 240.959777
+L 273.650614 239.87224
+L 273.955283 241.822901
+L 274.564619 240.407378
+L 275.173956 240.42464
+L 275.478624 241.650277
+L 276.087961 240.56274
+L 276.697297 240.182965
+L 277.001966 241.494914
+L 277.306634 241.546702
+L 277.611302 241.408602
+L 277.915971 241.667539
+L 278.220639 241.477652
+L 279.134644 241.374077
+L 279.439312 241.581227
+L 279.74398 240.372853
+L 280.048649 241.408602
+L 280.353317 241.408602
+L 280.657985 241.080614
+L 280.962654 240.062128
+L 281.267322 239.958553
+L 281.57199 241.909214
+L 281.876658 241.563964
+L 282.181327 240.856202
+L 282.485995 240.70084
+L 282.790663 240.735365
+L 283.095332 241.719326
+L 283.4 241.04609
+L 283.704668 240.07939
+L 284.009337 240.28654
+L 284.314005 240.131178
+L 284.618673 241.322289
+L 285.837346 241.339552
+L 286.142015 241.891951
+L 286.446683 241.978264
+L 287.05602 241.961001
+L 287.360688 240.925252
+L 287.665356 241.667539
+L 287.970025 241.598489
+L 288.274693 241.287764
+L 288.579361 240.35559
+L 288.884029 240.821677
+L 289.188698 242.012789
+L 289.798034 241.011565
+L 290.102703 240.804415
+L 290.407371 240.459165
+L 290.712039 241.667539
+L 291.016708 241.391339
+L 291.321376 240.56274
+L 291.626044 240.735365
+L 291.930713 240.70084
+L 292.235381 241.374077
+L 292.540049 241.443127
+L 293.149386 241.339552
+L 293.454054 241.408602
+L 293.758722 241.753851
+L 294.368059 241.667539
+L 294.977396 241.736589
+L 295.8914 241.736589
+L 296.500737 240.804415
+L 296.805405 242.047314
+L 297.110074 242.030051
+L 297.414742 241.753851
+L 298.024079 240.56274
+L 298.328747 242.064576
+L 298.938084 240.735365
+L 299.242752 240.528215
+L 299.54742 241.218714
+L 299.852088 241.270502
+L 300.156757 241.149664
+L 300.461425 241.322289
+L 301.070762 241.080614
+L 301.37543 241.546702
+L 302.289435 241.477652
+L 302.594103 241.097877
+L 302.898771 241.736589
+L 303.20344 241.753851
+L 303.508108 241.425864
+L 303.812776 240.459165
+L 304.422113 241.805639
+L 304.726781 241.891951
+L 305.03145 240.83894
+L 305.336118 240.83894
+L 305.640786 241.719326
+L 305.945455 241.684801
+L 306.250123 241.788376
+L 306.859459 240.407378
+L 307.164128 240.752627
+L 307.468796 242.047314
+L 307.773464 241.736589
+L 308.687469 241.702064
+L 308.992138 242.030051
+L 309.906143 242.047314
+L 310.210811 241.04609
+L 310.515479 242.064576
+L 311.124816 242.064576
+L 311.429484 241.149664
+L 311.734152 240.649052
+L 312.038821 241.995526
+L 312.343489 241.771114
+L 312.952826 241.080614
+L 313.257494 240.890727
+L 313.562162 241.978264
+L 313.86683 241.408602
+L 314.171499 240.476427
+L 314.780835 240.580002
+L 315.085504 241.097877
+L 315.999509 241.149664
+L 316.304177 241.166927
+L 316.608845 241.494914
+L 317.218182 241.529439
+L 319.046192 241.615752
+L 319.35086 241.063352
+L 319.655528 241.926476
+L 320.264865 241.978264
+L 320.569533 242.116364
+L 320.874201 240.90799
+L 321.17887 241.840164
+L 321.483538 241.857426
+L 321.788206 241.546702
+L 322.702211 241.270502
+L 323.00688 241.512177
+L 323.311548 241.477652
+L 323.616216 241.581227
+L 323.920885 241.374077
+L 324.225553 240.97704
+L 324.530221 241.563964
+L 324.834889 241.391339
+L 325.444226 241.339552
+L 325.748894 241.270502
+L 326.053563 241.581227
+L 326.358231 241.339552
+L 326.967568 241.339552
+L 327.272236 241.235977
+L 327.576904 241.650277
+L 328.186241 241.287764
+L 328.795577 241.305027
+L 329.100246 241.857426
+L 329.404914 241.356814
+L 330.014251 241.805639
+L 330.318919 241.115139
+L 330.623587 241.805639
+L 330.928256 241.270502
+L 331.537592 241.270502
+L 331.84226 241.080614
+L 332.146929 241.563964
+L 332.451597 241.425864
+L 332.756265 241.563964
+L 333.365602 241.115139
+L 333.67027 241.529439
+L 333.974939 241.287764
+L 334.279607 241.391339
+L 334.888943 241.166927
+L 335.193612 241.667539
+L 335.49828 241.253239
+L 336.107617 241.235977
+L 336.412285 241.063352
+L 336.716953 241.736589
+L 337.021622 241.322289
+L 337.32629 241.529439
+L 337.630958 241.563964
+L 337.935627 241.166927
+L 338.240295 241.477652
+L 338.544963 241.149664
+L 338.849631 241.512177
+L 339.458968 241.149664
+L 339.763636 241.615752
+L 340.068305 240.890727
+L 340.372973 241.529439
+L 340.677641 241.650277
+L 340.98231 241.460389
+L 341.286978 241.512177
+L 341.591646 241.166927
+L 341.896314 241.529439
+L 342.200983 241.633014
+L 342.505651 241.581227
+L 342.810319 241.840164
+L 343.114988 241.563964
+L 343.419656 241.805639
+L 343.724324 241.840164
+L 344.028993 241.287764
+L 344.333661 241.529439
+L 344.638329 241.270502
+L 344.942998 241.460389
+L 345.247666 241.097877
+L 345.857002 241.719326
+L 346.466339 241.408602
+L 346.771007 241.633014
+L 347.075676 241.719326
+L 347.380344 241.477652
+L 347.685012 241.753851
+L 347.989681 241.460389
+L 348.599017 241.581227
+L 348.903686 241.425864
+L 349.208354 241.633014
+L 349.513022 241.460389
+L 349.81769 241.460389
+L 350.122359 241.650277
+L 350.731695 241.615752
+L 351.341032 241.408602
+L 351.6457 241.909214
+L 351.950369 241.633014
+L 352.255037 241.753851
+L 352.559705 241.287764
+L 352.864373 241.097877
+L 353.169042 241.615752
+L 353.778378 241.512177
+L 354.083047 241.253239
+L 354.387715 241.356814
+L 354.692383 241.650277
+L 354.997052 241.391339
+L 355.606388 241.356814
+L 355.911057 241.270502
+L 356.215725 241.788376
+L 356.520393 241.201452
+L 357.12973 240.925252
+L 357.434398 241.166927
+L 357.739066 241.753851
+L 358.348403 241.235977
+L 358.653071 241.322289
+L 358.95774 241.028827
+L 359.262408 241.909214
+L 359.567076 241.529439
+L 359.871744 241.477652
+L 360.176413 241.598489
+L 360.481081 241.374077
+L 360.785749 241.633014
+L 361.090418 241.512177
+L 361.395086 241.581227
+L 361.699754 241.529439
+L 362.004423 241.753851
+L 362.309091 241.546702
+L 362.613759 241.702064
+L 362.918428 241.494914
+L 363.223096 241.443127
+L 363.527764 241.633014
+L 363.832432 241.598489
+L 364.137101 241.391339
+L 365.051106 241.633014
+L 365.965111 241.339552
+L 366.269779 241.650277
+L 366.574447 241.477652
+L 367.183784 241.512177
+L 367.79312 241.305027
+L 368.097789 241.408602
+L 368.402457 241.201452
+L 369.62113 241.063352
+L 369.925799 241.460389
+L 370.535135 241.132402
+L 370.839803 241.218714
+L 371.144472 240.873465
+L 371.44914 241.287764
+L 371.753808 241.270502
+L 372.058477 241.011565
+L 372.363145 241.063352
+L 372.667813 240.942515
+L 372.972482 241.235977
+L 373.27715 241.201452
+L 373.581818 241.028827
+L 373.581818 241.028827
+" clip-path="url(#pdceb6906fc)" style="fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square"/>
+ </g>
+ <g id="line2d_15">
+ <path d="M 69.218182 236.972143
+L 69.52285 236.903093
+L 69.827518 235.522094
+L 70.132187 235.901869
+L 70.741523 234.969694
+L 71.046192 235.677456
+L 71.35086 235.625669
+L 71.655528 238.629341
+L 71.960197 44.495424
+L 72.569533 44.478161
+L 73.17887 44.478161
+L 73.483538 236.298906
+L 73.788206 237.990629
+L 74.092875 238.007892
+L 74.397543 233.260708
+L 74.702211 233.571433
+L 75.00688 221.159706
+L 75.920885 229.152237
+L 76.225553 44.598999
+L 76.530221 44.443636
+L 78.662899 44.495424
+L 79.272236 44.443636
+L 79.576904 230.291561
+L 79.881572 44.478161
+L 80.795577 44.478161
+L 81.100246 44.460899
+L 81.404914 216.101797
+L 81.709582 218.328658
+L 82.014251 235.228632
+L 82.318919 44.478161
+L 83.232924 44.512686
+L 83.537592 241.166927
+L 83.84226 241.184189
+L 84.146929 241.425864
+L 84.756265 241.529439
+L 85.974939 241.408602
+L 86.584275 241.391339
+L 86.888943 241.132402
+L 88.107617 241.063352
+L 88.412285 241.512177
+L 88.716953 241.529439
+L 89.32629 241.408602
+L 90.849631 241.512177
+L 91.1543 241.080614
+L 91.458968 241.011565
+L 91.763636 241.115139
+L 92.068305 241.028827
+L 92.372973 241.149664
+L 93.591646 241.115139
+L 94.200983 241.04609
+L 94.505651 241.391339
+L 94.810319 241.063352
+L 95.724324 241.149664
+L 96.028993 241.04609
+L 96.333661 241.374077
+L 96.942998 241.529439
+L 97.247666 241.011565
+L 97.552334 241.132402
+L 97.857002 241.011565
+L 98.161671 241.028827
+L 98.466339 241.408602
+L 98.771007 241.425864
+L 99.075676 241.581227
+L 99.685012 241.443127
+L 101.81769 241.460389
+L 103.6457 241.460389
+L 105.47371 241.546702
+L 105.778378 241.425864
+L 107.30172 241.443127
+L 108.520393 241.391339
+L 109.12973 241.443127
+L 109.434398 241.581227
+L 110.043735 241.425864
+L 111.262408 241.425864
+L 111.567076 241.546702
+L 111.871744 241.546702
+L 112.481081 241.391339
+L 112.785749 44.926986
+L 113.090418 44.443636
+L 113.699754 44.478161
+L 114.004423 237.006668
+L 114.309091 236.644156
+L 114.613759 235.832819
+L 114.918428 236.782255
+L 115.527764 236.057231
+L 115.832432 236.81678
+L 116.137101 236.95488
+L 116.441769 239.043641
+L 116.746437 44.512686
+L 117.965111 44.478161
+L 118.269779 237.09298
+L 118.574447 238.076942
+L 118.879115 238.094204
+L 119.183784 233.864895
+L 119.488452 234.31372
+L 119.79312 222.040092
+L 120.097789 224.991978
+L 120.402457 225.924152
+L 120.707125 229.980836
+L 121.011794 44.581736
+L 121.316462 44.443636
+L 124.058477 44.495424
+L 124.363145 233.381546
+L 124.667813 44.460899
+L 125.886486 44.460899
+L 126.191155 216.18811
+L 126.495823 217.810783
+L 126.800491 235.901869
+L 127.10516 44.547211
+L 127.714496 44.460899
+L 128.019165 44.495424
+L 128.323833 238.991854
+L 128.628501 239.060904
+L 128.93317 241.477652
+L 129.237838 241.563964
+L 130.151843 241.494914
+L 131.370516 241.443127
+L 131.675184 239.147216
+L 131.979853 238.871016
+L 132.893857 238.991854
+L 133.198526 241.546702
+L 133.503194 241.581227
+L 133.807862 241.425864
+L 135.635872 241.563964
+L 135.940541 239.078166
+L 136.245209 238.940066
+L 136.549877 239.112691
+L 138.682555 239.043641
+L 138.987224 238.991854
+L 139.291892 241.529439
+L 139.59656 239.147216
+L 139.901229 238.940066
+L 140.205897 239.009116
+L 140.815233 239.268053
+L 141.119902 241.477652
+L 141.729238 241.598489
+L 142.033907 239.199004
+L 142.338575 239.216266
+L 142.643243 238.801966
+L 142.947912 239.526991
+L 143.25258 241.011565
+L 143.557248 241.080614
+L 143.861916 241.529439
+L 145.385258 241.460389
+L 145.994595 241.494914
+L 146.299263 241.563964
+L 146.603931 241.028827
+L 146.9086 240.90799
+L 147.517936 240.994302
+L 147.822604 241.028827
+L 148.127273 241.425864
+L 148.736609 241.581227
+L 150.564619 241.356814
+L 150.869287 240.942515
+L 151.173956 240.873465
+L 152.087961 240.97704
+L 153.915971 240.959777
+L 154.220639 241.563964
+L 154.525307 241.04609
+L 155.134644 240.90799
+L 155.74398 241.028827
+L 156.048649 241.443127
+L 156.657985 241.546702
+L 156.962654 241.011565
+L 157.267322 240.925252
+L 157.876658 240.372853
+L 158.181327 240.925252
+L 158.485995 241.028827
+L 158.790663 241.305027
+L 159.095332 241.270502
+L 159.4 241.408602
+L 159.704668 241.305027
+L 161.22801 241.477652
+L 161.532678 241.04609
+L 161.837346 241.028827
+L 162.142015 241.132402
+L 162.446683 241.115139
+L 162.751351 240.90799
+L 163.360688 241.460389
+L 163.970025 241.287764
+L 165.188698 241.460389
+L 165.493366 241.356814
+L 165.798034 240.97704
+L 166.102703 240.942515
+L 166.407371 241.115139
+L 167.016708 240.90799
+L 167.321376 240.97704
+L 167.930713 240.90799
+L 168.235381 241.063352
+L 168.844717 240.925252
+L 169.149386 241.443127
+L 169.454054 240.90799
+L 170.063391 241.063352
+L 170.368059 241.080614
+L 170.672727 240.97704
+L 170.977396 241.253239
+L 171.586732 241.477652
+L 171.8914 241.04609
+L 172.196069 241.097877
+L 172.500737 240.873465
+L 173.110074 241.253239
+L 176.461425 241.374077
+L 176.766093 241.235977
+L 178.898771 241.443127
+L 179.508108 241.322289
+L 180.726781 241.374077
+L 181.03145 241.218714
+L 181.640786 241.287764
+L 182.554791 241.218714
+L 183.773464 241.235977
+L 184.078133 241.443127
+L 184.687469 241.201452
+L 185.296806 241.201452
+L 185.906143 241.287764
+L 186.210811 241.425864
+L 186.515479 241.425864
+L 187.124816 241.253239
+L 187.429484 45.237711
+L 187.734152 44.668049
+L 188.343489 44.771624
+L 188.648157 236.488793
+L 188.952826 236.367956
+L 189.257494 235.660194
+L 189.562162 236.695943
+L 189.86683 236.730468
+L 190.171499 236.229856
+L 190.476167 236.799518
+L 190.780835 236.402481
+L 191.085504 237.679905
+L 191.390172 44.857936
+L 191.69484 44.737099
+L 191.999509 44.840674
+L 192.304177 44.788886
+L 192.608845 44.961511
+L 192.913514 237.524542
+L 193.218182 239.233528
+L 193.52285 239.129954
+L 193.827518 232.691046
+L 194.132187 233.329758
+L 194.436855 222.350817
+L 194.741523 225.371752
+L 195.046192 226.683701
+L 195.35086 230.136198
+L 195.655528 44.996036
+L 195.960197 44.668049
+L 196.264865 44.806149
+L 196.569533 44.685311
+L 197.788206 44.702574
+L 198.092875 44.909723
+L 198.397543 44.719836
+L 198.702211 44.754361
+L 199.00688 232.328534
+L 199.311548 44.771624
+L 199.616216 44.633524
+L 199.920885 44.633524
+L 200.530221 44.823411
+L 200.834889 215.463085
+L 201.139558 216.619672
+L 201.444226 234.917907
+L 201.748894 44.771624
+L 202.358231 44.737099
+L 202.662899 44.771624
+L 202.967568 241.201452
+L 204.490909 241.322289
+L 204.795577 241.218714
+L 205.404914 241.322289
+L 206.318919 241.356814
+L 206.623587 241.218714
+L 207.537592 241.356814
+L 210.584275 241.305027
+L 210.888943 241.184189
+L 211.802948 241.201452
+L 213.630958 241.218714
+L 213.935627 241.391339
+L 214.544963 241.201452
+L 215.458968 241.253239
+L 216.372973 241.443127
+L 216.98231 241.235977
+L 217.286978 240.90799
+L 217.896314 240.890727
+L 218.200983 241.04609
+L 218.505651 241.391339
+L 219.114988 241.322289
+L 219.419656 241.374077
+L 219.724324 241.235977
+L 220.333661 241.408602
+L 220.638329 241.270502
+L 220.942998 241.287764
+L 221.247666 240.942515
+L 221.552334 240.890727
+L 221.857002 241.011565
+L 222.466339 240.97704
+L 222.771007 241.408602
+L 223.075676 241.391339
+L 223.685012 241.184189
+L 224.294349 241.356814
+L 224.903686 241.253239
+L 225.208354 241.339552
+L 225.513022 240.994302
+L 225.81769 240.873465
+L 227.950369 240.90799
+L 228.559705 240.821677
+L 228.864373 241.201452
+L 229.169042 240.804415
+L 230.083047 240.873465
+L 230.387715 240.735365
+L 230.692383 241.115139
+L 230.997052 241.287764
+L 231.30172 241.305027
+L 231.606388 240.752627
+L 231.911057 240.856202
+L 232.520393 240.70084
+L 232.825061 241.028827
+L 233.434398 241.201452
+L 233.739066 241.115139
+L 234.348403 241.235977
+L 234.653071 241.132402
+L 234.95774 241.253239
+L 235.262408 241.166927
+L 235.871744 241.287764
+L 236.481081 241.080614
+L 238.613759 241.270502
+L 239.223096 241.132402
+L 239.832432 241.184189
+L 240.137101 241.132402
+L 240.441769 241.218714
+L 240.746437 241.028827
+L 241.355774 241.097877
+L 243.488452 241.028827
+L 243.79312 241.218714
+L 244.402457 240.994302
+L 245.316462 241.04609
+L 246.230467 241.184189
+L 246.839803 240.97704
+L 247.144472 240.614527
+L 247.44914 240.545477
+L 248.667813 241.184189
+L 249.581818 241.080614
+L 249.886486 241.149664
+L 250.495823 241.080614
+L 250.800491 241.080614
+L 251.10516 240.787152
+L 251.714496 240.718102
+L 252.019165 240.787152
+L 252.323833 240.718102
+L 252.628501 241.115139
+L 252.93317 241.201452
+L 253.237838 241.063352
+L 255.065848 241.201452
+L 255.370516 240.821677
+L 255.675184 240.735365
+L 255.979853 240.83894
+L 256.589189 240.718102
+L 257.807862 240.890727
+L 258.417199 240.76989
+L 258.721867 241.149664
+L 259.026536 240.821677
+L 259.331204 240.752627
+L 259.940541 240.873465
+L 260.245209 240.787152
+L 260.549877 241.063352
+L 261.159214 241.218714
+L 261.463882 240.76989
+L 261.76855 240.873465
+L 262.073219 44.978773
+L 262.377887 44.443636
+L 262.987224 44.460899
+L 263.291892 236.972143
+L 263.901229 235.763769
+L 264.205897 235.539356
+L 264.510565 235.746506
+L 264.815233 235.090532
+L 265.119902 236.039969
+L 265.42457 235.988181
+L 265.729238 237.973367
+L 266.033907 44.460899
+L 267.25258 44.478161
+L 267.557248 236.523318
+L 267.861916 237.904317
+L 268.166585 238.076942
+L 268.471253 234.676232
+L 268.775921 234.917907
+L 269.08059 223.058579
+L 269.385258 225.613427
+L 269.689926 226.735489
+L 269.994595 229.652849
+L 270.299263 44.581736
+L 270.603931 44.443636
+L 273.345946 44.443636
+L 273.650614 229.549274
+L 273.955283 44.512686
+L 275.173956 44.443636
+L 275.478624 216.015485
+L 275.783292 217.137546
+L 276.087961 234.900644
+L 276.392629 44.512686
+L 277.001966 44.460899
+L 277.306634 44.547211
+L 277.611302 241.011565
+L 277.915971 241.028827
+L 278.220639 241.322289
+L 280.657985 241.270502
+L 280.962654 240.994302
+L 281.876658 240.959777
+L 282.181327 241.028827
+L 282.485995 241.322289
+L 284.923342 241.270502
+L 285.532678 240.959777
+L 286.446683 240.97704
+L 288.274693 240.97704
+L 288.579361 241.287764
+L 289.493366 240.942515
+L 290.407371 241.115139
+L 290.712039 241.270502
+L 291.016708 241.287764
+L 291.321376 241.011565
+L 291.626044 240.97704
+L 291.930713 240.56274
+L 292.235381 240.441902
+L 292.540049 240.718102
+L 292.844717 240.735365
+L 293.149386 241.270502
+L 293.758722 241.097877
+L 295.586732 241.218714
+L 295.8914 240.752627
+L 296.500737 240.666315
+L 297.110074 240.70084
+L 297.414742 241.063352
+L 298.024079 241.115139
+L 298.328747 241.235977
+L 298.938084 241.080614
+L 299.242752 241.097877
+L 299.54742 241.235977
+L 300.461425 240.70084
+L 303.20344 240.718102
+L 303.508108 241.253239
+L 303.812776 240.752627
+L 304.422113 240.683577
+L 305.03145 240.752627
+L 305.336118 241.115139
+L 305.640786 241.270502
+L 305.945455 241.270502
+L 306.250123 240.76989
+L 306.554791 240.752627
+L 306.859459 44.926986
+L 307.164128 44.495424
+L 307.773464 44.495424
+L 308.078133 235.591144
+L 308.687469 234.17562
+L 308.992138 234.883382
+L 309.296806 234.900644
+L 309.601474 234.38277
+L 309.906143 235.245894
+L 310.210811 235.038744
+L 310.515479 237.16203
+L 310.820147 44.529949
+L 311.429484 44.529949
+L 312.038821 44.547211
+L 312.343489 235.936394
+L 312.648157 237.43823
+L 312.952826 237.835267
+L 313.257494 232.345797
+L 313.562162 232.760096
+L 313.86683 220.572781
+L 314.780835 226.942638
+L 315.085504 44.616261
+L 315.390172 44.495424
+L 318.132187 44.478161
+L 318.436855 230.947535
+L 318.741523 44.512686
+L 319.655528 44.495424
+L 319.960197 44.512686
+L 320.264865 215.91191
+L 320.569533 217.914358
+L 320.874201 234.330982
+L 321.17887 44.581736
+L 321.788206 44.512686
+L 322.092875 44.529949
+L 322.397543 241.097877
+L 323.311548 241.235977
+L 323.920885 241.235977
+L 324.834889 241.218714
+L 325.444226 241.322289
+L 326.053563 241.201452
+L 327.272236 241.218714
+L 327.881572 241.322289
+L 328.490909 241.253239
+L 329.100246 241.235977
+L 330.928256 241.166927
+L 331.537592 241.097877
+L 333.67027 241.132402
+L 334.888943 241.149664
+L 335.802948 241.270502
+L 336.412285 241.115139
+L 336.716953 237.78348
+L 337.021622 237.30013
+L 337.630958 237.541805
+L 337.935627 241.132402
+L 339.1543 241.166927
+L 340.068305 241.218714
+L 340.372973 241.132402
+L 340.677641 237.593592
+L 340.98231 237.455492
+L 341.591646 237.490017
+L 341.896314 237.731692
+L 342.200983 241.097877
+L 342.505651 241.305027
+L 343.114988 241.097877
+L 344.028993 241.235977
+L 344.333661 241.374077
+L 344.638329 241.235977
+L 344.942998 237.731692
+L 345.247666 237.541805
+L 345.552334 237.662642
+L 345.857002 237.403705
+L 347.380344 237.593592
+L 347.685012 237.628117
+L 347.989681 237.50728
+L 348.294349 241.322289
+L 348.599017 237.64538
+L 349.208354 237.43823
+L 349.513022 237.541805
+L 349.81769 237.800742
+L 350.122359 241.184189
+L 350.731695 241.391339
+L 351.036364 237.904317
+L 351.341032 237.766217
+L 351.6457 237.351917
+L 351.950369 238.059679
+L 352.255037 241.04609
+L 352.559705 241.04609
+L 352.864373 241.305027
+L 353.47371 241.391339
+L 354.083047 241.322289
+L 355.30172 241.287764
+L 355.606388 241.011565
+L 356.215725 240.97704
+L 356.520393 241.011565
+L 356.825061 240.90799
+L 357.12973 241.356814
+L 357.434398 241.374077
+L 357.739066 241.253239
+L 359.262408 241.356814
+L 359.567076 241.408602
+L 359.871744 240.994302
+L 360.176413 240.925252
+L 360.481081 241.04609
+L 360.785749 240.890727
+L 361.395086 240.925252
+L 361.699754 240.873465
+L 362.309091 240.942515
+L 362.918428 240.856202
+L 363.223096 241.201452
+L 363.527764 240.890727
+L 364.746437 240.873465
+L 365.051106 241.132402
+L 365.660442 241.287764
+L 365.965111 240.856202
+L 366.269779 240.97704
+L 366.574447 240.545477
+L 366.879115 240.649052
+L 367.183784 240.614527
+L 368.402457 241.218714
+L 368.707125 241.132402
+L 369.316462 241.287764
+L 369.62113 241.149664
+L 369.925799 241.201452
+L 370.230467 241.132402
+L 370.535135 240.683577
+L 370.839803 240.597265
+L 371.144472 240.735365
+L 371.753808 240.597265
+L 372.058477 241.028827
+L 372.363145 241.201452
+L 372.667813 241.063352
+L 373.27715 241.097877
+L 373.581818 241.184189
+L 373.581818 241.184189
+" clip-path="url(#pdceb6906fc)" style="fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: square"/>
+ </g>
+ <g id="patch_3">
+ <path d="M 54 252
+L 54 34.56
+" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
+ </g>
+ <g id="patch_4">
+ <path d="M 388.8 252
+L 388.8 34.56
+" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
+ </g>
+ <g id="patch_5">
+ <path d="M 54 252
+L 388.8 252
+" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
+ </g>
+ <g id="patch_6">
+ <path d="M 54 34.56
+L 388.8 34.56
+" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
+ </g>
+ <g id="text_15">
+ <!-- Parameter Values -->
+ <g transform="translate(168.675 28.56)scale(0.12 -0.12)">
+ <defs>
+ <path id="DejaVuSans-56" d="M 1831 0
+L 50 4666
+L 709 4666
+L 2188 738
+L 3669 4666
+L 4325 4666
+L 2547 0
+L 1831 0
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-6c" d="M 603 4863
+L 1178 4863
+L 1178 0
+L 603 0
+L 603 4863
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-75" d="M 544 1381
+L 544 3500
+L 1119 3500
+L 1119 1403
+Q 1119 906 1312 657
+Q 1506 409 1894 409
+Q 2359 409 2629 706
+Q 2900 1003 2900 1516
+L 2900 3500
+L 3475 3500
+L 3475 0
+L 2900 0
+L 2900 538
+Q 2691 219 2414 64
+Q 2138 -91 1772 -91
+Q 1169 -91 856 284
+Q 544 659 544 1381
+z
+M 1991 3584
+L 1991 3584
+z
+" transform="scale(0.015625)"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-50"/>
+ <use xlink:href="#DejaVuSans-61" x="55.802734"/>
+ <use xlink:href="#DejaVuSans-72" x="117.082031"/>
+ <use xlink:href="#DejaVuSans-61" x="158.195312"/>
+ <use xlink:href="#DejaVuSans-6d" x="219.474609"/>
+ <use xlink:href="#DejaVuSans-65" x="316.886719"/>
+ <use xlink:href="#DejaVuSans-74" x="378.410156"/>
+ <use xlink:href="#DejaVuSans-65" x="417.619141"/>
+ <use xlink:href="#DejaVuSans-72" x="479.142578"/>
+ <use xlink:href="#DejaVuSans-20" x="520.255859"/>
+ <use xlink:href="#DejaVuSans-56" x="552.042969"/>
+ <use xlink:href="#DejaVuSans-61" x="612.701172"/>
+ <use xlink:href="#DejaVuSans-6c" x="673.980469"/>
+ <use xlink:href="#DejaVuSans-75" x="701.763672"/>
+ <use xlink:href="#DejaVuSans-65" x="765.142578"/>
+ <use xlink:href="#DejaVuSans-73" x="826.666016"/>
+ </g>
+ </g>
+ <g id="legend_1">
+ <g id="patch_7">
+ <path d="M 276.029688 159.458125
+L 381.8 159.458125
+Q 383.8 159.458125 383.8 157.458125
+L 383.8 129.101875
+Q 383.8 127.101875 381.8 127.101875
+L 276.029688 127.101875
+Q 274.029688 127.101875 274.029688 129.101875
+L 274.029688 157.458125
+Q 274.029688 159.458125 276.029688 159.458125
+z
+" style="fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter"/>
+ </g>
+ <g id="line2d_16">
+ <path d="M 278.029688 135.200312
+L 288.029688 135.200312
+L 298.029688 135.200312
+" style="fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square"/>
+ </g>
+ <g id="text_16">
+ <!-- top-10% -->
+ <g transform="translate(306.029688 138.700312)scale(0.1 -0.1)">
+ <defs>
+ <path id="DejaVuSans-6f" d="M 1959 3097
+Q 1497 3097 1228 2736
+Q 959 2375 959 1747
+Q 959 1119 1226 758
+Q 1494 397 1959 397
+Q 2419 397 2687 759
+Q 2956 1122 2956 1747
+Q 2956 2369 2687 2733
+Q 2419 3097 1959 3097
+z
+M 1959 3584
+Q 2709 3584 3137 3096
+Q 3566 2609 3566 1747
+Q 3566 888 3137 398
+Q 2709 -91 1959 -91
+Q 1206 -91 779 398
+Q 353 888 353 1747
+Q 353 2609 779 3096
+Q 1206 3584 1959 3584
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-70" d="M 1159 525
+L 1159 -1331
+L 581 -1331
+L 581 3500
+L 1159 3500
+L 1159 2969
+Q 1341 3281 1617 3432
+Q 1894 3584 2278 3584
+Q 2916 3584 3314 3078
+Q 3713 2572 3713 1747
+Q 3713 922 3314 415
+Q 2916 -91 2278 -91
+Q 1894 -91 1617 61
+Q 1341 213 1159 525
+z
+M 3116 1747
+Q 3116 2381 2855 2742
+Q 2594 3103 2138 3103
+Q 1681 3103 1420 2742
+Q 1159 2381 1159 1747
+Q 1159 1113 1420 752
+Q 1681 391 2138 391
+Q 2594 391 2855 752
+Q 3116 1113 3116 1747
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-2d" d="M 313 2009
+L 1997 2009
+L 1997 1497
+L 313 1497
+L 313 2009
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-25" d="M 4653 2053
+Q 4381 2053 4226 1822
+Q 4072 1591 4072 1178
+Q 4072 772 4226 539
+Q 4381 306 4653 306
+Q 4919 306 5073 539
+Q 5228 772 5228 1178
+Q 5228 1588 5073 1820
+Q 4919 2053 4653 2053
+z
+M 4653 2450
+Q 5147 2450 5437 2106
+Q 5728 1763 5728 1178
+Q 5728 594 5436 251
+Q 5144 -91 4653 -91
+Q 4153 -91 3862 251
+Q 3572 594 3572 1178
+Q 3572 1766 3864 2108
+Q 4156 2450 4653 2450
+z
+M 1428 4353
+Q 1159 4353 1004 4120
+Q 850 3888 850 3481
+Q 850 3069 1003 2837
+Q 1156 2606 1428 2606
+Q 1700 2606 1854 2837
+Q 2009 3069 2009 3481
+Q 2009 3884 1853 4118
+Q 1697 4353 1428 4353
+z
+M 4250 4750
+L 4750 4750
+L 1831 -91
+L 1331 -91
+L 4250 4750
+z
+M 1428 4750
+Q 1922 4750 2215 4408
+Q 2509 4066 2509 3481
+Q 2509 2891 2217 2550
+Q 1925 2209 1428 2209
+Q 931 2209 642 2551
+Q 353 2894 353 3481
+Q 353 4063 643 4406
+Q 934 4750 1428 4750
+z
+" transform="scale(0.015625)"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-74"/>
+ <use xlink:href="#DejaVuSans-6f" x="39.208984"/>
+ <use xlink:href="#DejaVuSans-70" x="100.390625"/>
+ <use xlink:href="#DejaVuSans-2d" x="163.867188"/>
+ <use xlink:href="#DejaVuSans-31" x="199.951172"/>
+ <use xlink:href="#DejaVuSans-30" x="263.574219"/>
+ <use xlink:href="#DejaVuSans-25" x="327.197266"/>
+ </g>
+ </g>
+ <g id="line2d_17">
+ <path d="M 278.029688 149.878437
+L 288.029688 149.878437
+L 298.029688 149.878437
+" style="fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: square"/>
+ </g>
+ <g id="text_17">
+ <!-- Sym2 top-10% -->
+ <g transform="translate(306.029688 153.378437)scale(0.1 -0.1)">
+ <defs>
+ <path id="DejaVuSans-53" d="M 3425 4513
+L 3425 3897
+Q 3066 4069 2747 4153
+Q 2428 4238 2131 4238
+Q 1616 4238 1336 4038
+Q 1056 3838 1056 3469
+Q 1056 3159 1242 3001
+Q 1428 2844 1947 2747
+L 2328 2669
+Q 3034 2534 3370 2195
+Q 3706 1856 3706 1288
+Q 3706 609 3251 259
+Q 2797 -91 1919 -91
+Q 1588 -91 1214 -16
+Q 841 59 441 206
+L 441 856
+Q 825 641 1194 531
+Q 1563 422 1919 422
+Q 2459 422 2753 634
+Q 3047 847 3047 1241
+Q 3047 1584 2836 1778
+Q 2625 1972 2144 2069
+L 1759 2144
+Q 1053 2284 737 2584
+Q 422 2884 422 3419
+Q 422 4038 858 4394
+Q 1294 4750 2059 4750
+Q 2388 4750 2728 4690
+Q 3069 4631 3425 4513
+z
+" transform="scale(0.015625)"/>
+ <path id="DejaVuSans-79" d="M 2059 -325
+Q 1816 -950 1584 -1140
+Q 1353 -1331 966 -1331
+L 506 -1331
+L 506 -850
+L 844 -850
+Q 1081 -850 1212 -737
+Q 1344 -625 1503 -206
+L 1606 56
+L 191 3500
+L 800 3500
+L 1894 763
+L 2988 3500
+L 3597 3500
+L 2059 -325
+z
+" transform="scale(0.015625)"/>
+ </defs>
+ <use xlink:href="#DejaVuSans-53"/>
+ <use xlink:href="#DejaVuSans-79" x="63.476562"/>
+ <use xlink:href="#DejaVuSans-6d" x="122.65625"/>
+ <use xlink:href="#DejaVuSans-32" x="220.068359"/>
+ <use xlink:href="#DejaVuSans-20" x="283.691406"/>
+ <use xlink:href="#DejaVuSans-74" x="315.478516"/>
+ <use xlink:href="#DejaVuSans-6f" x="354.6875"/>
+ <use xlink:href="#DejaVuSans-70" x="415.869141"/>
+ <use xlink:href="#DejaVuSans-2d" x="479.345703"/>
+ <use xlink:href="#DejaVuSans-31" x="515.429688"/>
+ <use xlink:href="#DejaVuSans-30" x="579.052734"/>
+ <use xlink:href="#DejaVuSans-25" x="642.675781"/>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ <defs>
+ <clipPath id="pdceb6906fc">
+ <rect x="54" y="34.56" width="334.8" height="217.44"/>
+ </clipPath>
+ </defs>
+</svg>
diff --git a/random files/ParametersWaveletHaarAcc.png b/random files/ParametersWaveletHaarAcc.png
new file mode 100644
index 0000000000000000000000000000000000000000..08c1ba7cc2068fbe3dca1317f50aab8ec4ad0483
Binary files /dev/null and b/random files/ParametersWaveletHaarAcc.png differ
diff --git a/random files/PartialModel.py b/random files/PartialModel.py
new file mode 100644
index 0000000000000000000000000000000000000000..5fccc99b48836ba361da3bcf390d561a036e7a01
--- /dev/null
+++ b/random files/PartialModel.py
@@ -0,0 +1,381 @@
+import json
+import logging
+import os
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from decentralizepy.sharing.Sharing import Sharing
+from decentralizepy.utils import conditional_value, identity
+
+
+class PartialModel(Sharing):
+ """
+ This class implements the vanilla version of partial model sharing.
+
+ """
+
+ def __init__(
+ self,
+ rank,
+ machine_id,
+ communication,
+ mapping,
+ graph,
+ model,
+ dataset,
+ log_dir,
+ alpha=1.0,
+ dict_ordered=True,
+ save_shared=False,
+ metadata_cap=1.0,
+ accumulation=False,
+ save_accumulated="",
+ change_transformer=identity,
+ accumulate_averaging_changes=False,
+ ):
+ """
+ Constructor
+
+ Parameters
+ ----------
+ rank : int
+ Local rank
+ machine_id : int
+ Global machine id
+ communication : decentralizepy.communication.Communication
+ Communication module used to send and receive messages
+ mapping : decentralizepy.mappings.Mapping
+ Mapping (rank, machine_id) -> uid
+ graph : decentralizepy.graphs.Graph
+ Graph reprensenting neighbors
+ model : decentralizepy.models.Model
+ Model to train
+ dataset : decentralizepy.datasets.Dataset
+ Dataset for sharing data. Not implemented yet! TODO
+ log_dir : str
+ Location to write shared_params (only writing for 2 procs per machine)
+ alpha : float
+ Percentage of model to share
+ dict_ordered : bool
+ Specifies if the python dict maintains the order of insertion
+ save_shared : bool
+ Specifies if the indices of shared parameters should be logged
+ metadata_cap : float
+ Share full model when self.alpha > metadata_cap
+ accumulation : bool
+ True if the the indices to share should be selected based on accumulated frequency change
+ save_accumulated : bool
+ True if accumulated weight change should be written to file. In case of accumulation the accumulated change
+ is stored. If a change_transformer is used then the transformed change is stored.
+ change_transformer : (x: Tensor) -> Tensor
+ A function that transforms the model change into other domains. Default: identity function
+ accumulate_averaging_changes: bool
+ True if the accumulation should account the model change due to averaging
+
+ """
+ super().__init__(
+ rank, machine_id, communication, mapping, graph, model, dataset, log_dir
+ )
+ self.alpha = alpha
+ self.dict_ordered = dict_ordered
+ self.save_shared = save_shared
+ self.metadata_cap = metadata_cap
+ self.total_meta = 0
+ self.accumulation = accumulation
+ self.save_accumulated = conditional_value(save_accumulated, "", False)
+ self.change_transformer = change_transformer
+ self.accumulate_averaging_changes = accumulate_averaging_changes
+
+ # getting the initial model
+ self.shapes = []
+ self.lens = []
+ with torch.no_grad():
+ tensors_to_cat = []
+ for _, v in self.model.state_dict().items():
+ self.shapes.append(v.shape)
+ t = v.flatten()
+ self.lens.append(t.shape[0])
+ tensors_to_cat.append(t)
+ self.init_model = torch.cat(tensors_to_cat, dim=0)
+ if self.accumulation:
+ self.model.accumulated_changes = torch.zeros_like(
+ self.change_transformer(self.init_model)
+ )
+ self.prev = self.init_model
+
+ if self.save_accumulated:
+ self.model_change_path = os.path.join(
+ self.log_dir, "model_change/{}".format(self.rank)
+ )
+ Path(self.model_change_path).mkdir(parents=True, exist_ok=True)
+
+ self.model_val_path = os.path.join(
+ self.log_dir, "model_val/{}".format(self.rank)
+ )
+ Path(self.model_val_path).mkdir(parents=True, exist_ok=True)
+
+ # Only save for 2 procs: Save space
+ if self.save_shared and not (rank == 0 or rank == 1):
+ self.save_shared = False
+
+ if self.save_shared:
+ self.folder_path = os.path.join(
+ self.log_dir, "shared_params/{}".format(self.rank)
+ )
+ Path(self.folder_path).mkdir(parents=True, exist_ok=True)
+
+ self.model.shared_parameters_counter = torch.zeros(
+ self.change_transformer(self.init_model).shape[0], dtype=torch.int32
+ )
+
+ self.caches = dict()
+ my_neighbors = self.graph.neighbors(self.uid)
+ for n in my_neighbors:
+ self.caches[n] = self.init_model.clone().detach()
+ self.my_uid = self.mapping.get_uid(self.rank, self.machine_id)
+ self.caches[self.my_uid] = self.init_model.clone().detach()
+ self.e = torch.zeros_like(self.init_model)
+
+ def extract_top_gradients(self):
+ """
+ Extract the indices and values of the topK gradients.
+ The gradients must have been accumulated.
+
+ Returns
+ -------
+ tuple
+ (a,b). a: The magnitudes of the topK gradients, b: Their indices.
+
+ """
+
+ logging.info("Returning topk gradients")
+ G_topk = torch.abs(self.u)
+ std, mean = torch.std_mean(G_topk, unbiased=False)
+ self.std = std.item()
+ self.mean = mean.item()
+ return torch.topk(
+ G_topk, round(self.alpha * G_topk.shape[0]), dim=0, sorted=False
+ )
+
+ def serialized_model(self):
+ """
+ Convert model to a dict. self.alpha specifies the fraction of model to send.
+
+ Returns
+ -------
+ dict
+ Model converted to a dict
+
+ """
+ if self.alpha >= self.metadata_cap: # Share fully
+ return super().serialized_model()
+
+ with torch.no_grad():
+ _, G_topk = self.extract_top_gradients()
+ self.model.shared_parameters_counter[G_topk] += 1
+
+ start_index = 0
+ std_dict = {}
+ new_model = self.init_model.clone().detach()
+ new_model[G_topk] += self.u[G_topk]
+ self.caches[self.my_uid] = new_model.clone().detach()
+ for i, key in enumerate(self.model.state_dict()):
+ end_index = start_index + self.lens[i]
+ std_dict[key] = new_model[start_index:end_index].reshape(self.shapes[i])
+ start_index = end_index
+
+ self.model.load_state_dict(std_dict)
+ self.g[G_topk] -= self.g[G_topk]
+ self.e = self.g
+
+ if self.save_shared:
+ shared_params = dict()
+ shared_params["order"] = list(self.model.state_dict().keys())
+ shapes = dict()
+ for k, v in self.model.state_dict().items():
+ shapes[k] = list(v.shape)
+ shared_params["shapes"] = shapes
+
+ shared_params[self.communication_round] = G_topk.tolist()
+
+ with open(
+ os.path.join(
+ self.folder_path,
+ "{}_shared_params.json".format(self.communication_round + 1),
+ ),
+ "w",
+ ) as of:
+ json.dump(shared_params, of)
+
+ logging.info("Extracting topk params")
+
+ logging.info("Generating dictionary to send")
+
+ m = dict()
+
+ if not self.dict_ordered:
+ raise NotImplementedError
+
+ m["indices"] = G_topk.numpy().astype(np.int32)
+
+ m["params"] = self.u[G_topk].numpy()
+
+ m["send_partial"] = True
+
+ assert len(m["indices"]) == len(m["params"])
+ logging.info("Elements sending: {}".format(len(m["indices"])))
+
+ logging.info("Generated dictionary to send")
+
+ logging.info("Converted dictionary to pickle")
+ self.total_data += len(self.communication.encrypt(m["params"]))
+ self.total_meta += len(self.communication.encrypt(m["indices"]))
+
+ return m
+
+ def deserialized_model(self, m):
+ """
+ Convert received dict to state_dict.
+
+ Parameters
+ ----------
+ m : dict
+ dict received
+
+ Returns
+ -------
+ state_dict
+ state_dict of received
+
+ """
+ if "send_partial" not in m:
+ return super().deserialized_model(m)
+
+ with torch.no_grad():
+ state_dict = self.model.state_dict()
+
+ if not self.dict_ordered:
+ raise NotImplementedError
+
+ index_tensor = torch.tensor(m["indices"], dtype=torch.long)
+ values = torch.tensor(m["params"])
+ return index_tensor, values
+
+ def _pre_step(self):
+ """
+ Called at the beginning of step.
+
+ """
+ tensors_to_cat = [v.data.flatten() for _, v in self.model.state_dict().items()]
+ self.post_train_model = torch.cat(tensors_to_cat, dim=0)
+ self.g = self.e + (self.post_train_model - self.init_model)
+
+ def _averaging(self):
+ """
+ Averages the received model with the local model
+
+ """
+ with torch.no_grad():
+ total = torch.zeros_like(self.init_model)
+ weight_total = 0
+ for i, n in enumerate(self.caches):
+ if n != self.my_uid:
+ data = self.caches[n]
+ degree = len(self.graph.neighbors(n))
+ weight = 1 / (
+ max(len(self.peer_deques), degree) + 1
+ ) # Metro-Hastings
+ weight_total += weight
+ total += data * weight
+
+ total += (1 - weight_total) * self.caches[self.my_uid] # Metro-Hastings
+ self.avg = total
+
+ def step(self):
+ """
+ Perform a sharing step. Implements D-PSGD.
+
+ """
+
+ self._pre_step()
+
+ logging.info("Starting model averaging after receiving from all neighbors")
+ self._averaging()
+ logging.info("Model averaging complete")
+
+ self.u = (self.avg - self.init_model) - self.g
+
+ data = self.serialized_model()
+ my_uid = self.mapping.get_uid(self.rank, self.machine_id)
+ all_neighbors = self.graph.neighbors(my_uid)
+ iter_neighbors = self.get_neighbors(all_neighbors)
+ data["degree"] = len(all_neighbors)
+ data["iteration"] = self.communication_round
+ for neighbor in iter_neighbors:
+ self.communication.send(neighbor, data)
+
+ logging.info("Waiting for messages from neighbors")
+ while not self.received_from_all():
+ sender, data = self.communication.receive()
+ logging.debug("Received model from {}".format(sender))
+ degree = data["degree"]
+ iteration = data["iteration"]
+ del data["degree"]
+ del data["iteration"]
+ self.peer_deques[sender].append((degree, iteration, data))
+ logging.info(
+ "Deserialized received model from {} of iteration {}".format(
+ sender, iteration
+ )
+ )
+
+ for i, n in enumerate(self.peer_deques):
+ degree, iteration, data = self.peer_deques[n].popleft()
+ ind, val = self.deserialized_model(data)
+ self.caches[n][ind] += val
+
+ self.communication_round += 1
+
+ tensors_to_cat = [v.data.flatten() for _, v in self.model.state_dict().items()]
+ post_share_model = torch.cat(tensors_to_cat, dim=0)
+ self.init_model = post_share_model
+ self._post_step()
+
+ def save_vector(self, v, s):
+ """
+ Saves the given vector to the file.
+
+ Parameters
+ ----------
+ v : torch.tensor
+ The torch tensor to write to file
+ s : str
+ Path to folder to write to
+
+ """
+ output_dict = dict()
+ output_dict["order"] = list(self.model.state_dict().keys())
+ shapes = dict()
+ for k, v1 in self.model.state_dict().items():
+ shapes[k] = list(v1.shape)
+ output_dict["shapes"] = shapes
+
+ output_dict["tensor"] = v.tolist()
+
+ with open(
+ os.path.join(
+ s,
+ "{}.json".format(self.communication_round + 1),
+ ),
+ "w",
+ ) as of:
+ json.dump(output_dict, of)
+
+ def save_change(self):
+ """
+ Saves the change and the gradient values for every iteration
+
+ """
+ self.save_vector(self.model.model_change, self.model_change_path)
diff --git a/random files/PartialModelWed.py b/random files/PartialModelWed.py
new file mode 100644
index 0000000000000000000000000000000000000000..5fccc99b48836ba361da3bcf390d561a036e7a01
--- /dev/null
+++ b/random files/PartialModelWed.py
@@ -0,0 +1,381 @@
+import json
+import logging
+import os
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from decentralizepy.sharing.Sharing import Sharing
+from decentralizepy.utils import conditional_value, identity
+
+
+class PartialModel(Sharing):
+ """
+ This class implements the vanilla version of partial model sharing.
+
+ """
+
+ def __init__(
+ self,
+ rank,
+ machine_id,
+ communication,
+ mapping,
+ graph,
+ model,
+ dataset,
+ log_dir,
+ alpha=1.0,
+ dict_ordered=True,
+ save_shared=False,
+ metadata_cap=1.0,
+ accumulation=False,
+ save_accumulated="",
+ change_transformer=identity,
+ accumulate_averaging_changes=False,
+ ):
+ """
+ Constructor
+
+ Parameters
+ ----------
+ rank : int
+ Local rank
+ machine_id : int
+ Global machine id
+ communication : decentralizepy.communication.Communication
+ Communication module used to send and receive messages
+ mapping : decentralizepy.mappings.Mapping
+ Mapping (rank, machine_id) -> uid
+ graph : decentralizepy.graphs.Graph
+ Graph reprensenting neighbors
+ model : decentralizepy.models.Model
+ Model to train
+ dataset : decentralizepy.datasets.Dataset
+ Dataset for sharing data. Not implemented yet! TODO
+ log_dir : str
+ Location to write shared_params (only writing for 2 procs per machine)
+ alpha : float
+ Percentage of model to share
+ dict_ordered : bool
+ Specifies if the python dict maintains the order of insertion
+ save_shared : bool
+ Specifies if the indices of shared parameters should be logged
+ metadata_cap : float
+ Share full model when self.alpha > metadata_cap
+ accumulation : bool
+ True if the the indices to share should be selected based on accumulated frequency change
+ save_accumulated : bool
+ True if accumulated weight change should be written to file. In case of accumulation the accumulated change
+ is stored. If a change_transformer is used then the transformed change is stored.
+ change_transformer : (x: Tensor) -> Tensor
+ A function that transforms the model change into other domains. Default: identity function
+ accumulate_averaging_changes: bool
+ True if the accumulation should account the model change due to averaging
+
+ """
+ super().__init__(
+ rank, machine_id, communication, mapping, graph, model, dataset, log_dir
+ )
+ self.alpha = alpha
+ self.dict_ordered = dict_ordered
+ self.save_shared = save_shared
+ self.metadata_cap = metadata_cap
+ self.total_meta = 0
+ self.accumulation = accumulation
+ self.save_accumulated = conditional_value(save_accumulated, "", False)
+ self.change_transformer = change_transformer
+ self.accumulate_averaging_changes = accumulate_averaging_changes
+
+ # getting the initial model
+ self.shapes = []
+ self.lens = []
+ with torch.no_grad():
+ tensors_to_cat = []
+ for _, v in self.model.state_dict().items():
+ self.shapes.append(v.shape)
+ t = v.flatten()
+ self.lens.append(t.shape[0])
+ tensors_to_cat.append(t)
+ self.init_model = torch.cat(tensors_to_cat, dim=0)
+ if self.accumulation:
+ self.model.accumulated_changes = torch.zeros_like(
+ self.change_transformer(self.init_model)
+ )
+ self.prev = self.init_model
+
+ if self.save_accumulated:
+ self.model_change_path = os.path.join(
+ self.log_dir, "model_change/{}".format(self.rank)
+ )
+ Path(self.model_change_path).mkdir(parents=True, exist_ok=True)
+
+ self.model_val_path = os.path.join(
+ self.log_dir, "model_val/{}".format(self.rank)
+ )
+ Path(self.model_val_path).mkdir(parents=True, exist_ok=True)
+
+ # Only save for 2 procs: Save space
+ if self.save_shared and not (rank == 0 or rank == 1):
+ self.save_shared = False
+
+ if self.save_shared:
+ self.folder_path = os.path.join(
+ self.log_dir, "shared_params/{}".format(self.rank)
+ )
+ Path(self.folder_path).mkdir(parents=True, exist_ok=True)
+
+ self.model.shared_parameters_counter = torch.zeros(
+ self.change_transformer(self.init_model).shape[0], dtype=torch.int32
+ )
+
+ self.caches = dict()
+ my_neighbors = self.graph.neighbors(self.uid)
+ for n in my_neighbors:
+ self.caches[n] = self.init_model.clone().detach()
+ self.my_uid = self.mapping.get_uid(self.rank, self.machine_id)
+ self.caches[self.my_uid] = self.init_model.clone().detach()
+ self.e = torch.zeros_like(self.init_model)
+
+ def extract_top_gradients(self):
+ """
+ Extract the indices and values of the topK gradients.
+ The gradients must have been accumulated.
+
+ Returns
+ -------
+ tuple
+ (a,b). a: The magnitudes of the topK gradients, b: Their indices.
+
+ """
+
+ logging.info("Returning topk gradients")
+ G_topk = torch.abs(self.u)
+ std, mean = torch.std_mean(G_topk, unbiased=False)
+ self.std = std.item()
+ self.mean = mean.item()
+ return torch.topk(
+ G_topk, round(self.alpha * G_topk.shape[0]), dim=0, sorted=False
+ )
+
+ def serialized_model(self):
+ """
+ Convert model to a dict. self.alpha specifies the fraction of model to send.
+
+ Returns
+ -------
+ dict
+ Model converted to a dict
+
+ """
+ if self.alpha >= self.metadata_cap: # Share fully
+ return super().serialized_model()
+
+ with torch.no_grad():
+ _, G_topk = self.extract_top_gradients()
+ self.model.shared_parameters_counter[G_topk] += 1
+
+ start_index = 0
+ std_dict = {}
+ new_model = self.init_model.clone().detach()
+ new_model[G_topk] += self.u[G_topk]
+ self.caches[self.my_uid] = new_model.clone().detach()
+ for i, key in enumerate(self.model.state_dict()):
+ end_index = start_index + self.lens[i]
+ std_dict[key] = new_model[start_index:end_index].reshape(self.shapes[i])
+ start_index = end_index
+
+ self.model.load_state_dict(std_dict)
+ self.g[G_topk] -= self.g[G_topk]
+ self.e = self.g
+
+ if self.save_shared:
+ shared_params = dict()
+ shared_params["order"] = list(self.model.state_dict().keys())
+ shapes = dict()
+ for k, v in self.model.state_dict().items():
+ shapes[k] = list(v.shape)
+ shared_params["shapes"] = shapes
+
+ shared_params[self.communication_round] = G_topk.tolist()
+
+ with open(
+ os.path.join(
+ self.folder_path,
+ "{}_shared_params.json".format(self.communication_round + 1),
+ ),
+ "w",
+ ) as of:
+ json.dump(shared_params, of)
+
+ logging.info("Extracting topk params")
+
+ logging.info("Generating dictionary to send")
+
+ m = dict()
+
+ if not self.dict_ordered:
+ raise NotImplementedError
+
+ m["indices"] = G_topk.numpy().astype(np.int32)
+
+ m["params"] = self.u[G_topk].numpy()
+
+ m["send_partial"] = True
+
+ assert len(m["indices"]) == len(m["params"])
+ logging.info("Elements sending: {}".format(len(m["indices"])))
+
+ logging.info("Generated dictionary to send")
+
+ logging.info("Converted dictionary to pickle")
+ self.total_data += len(self.communication.encrypt(m["params"]))
+ self.total_meta += len(self.communication.encrypt(m["indices"]))
+
+ return m
+
+ def deserialized_model(self, m):
+ """
+ Convert received dict to state_dict.
+
+ Parameters
+ ----------
+ m : dict
+ dict received
+
+ Returns
+ -------
+ state_dict
+ state_dict of received
+
+ """
+ if "send_partial" not in m:
+ return super().deserialized_model(m)
+
+ with torch.no_grad():
+ state_dict = self.model.state_dict()
+
+ if not self.dict_ordered:
+ raise NotImplementedError
+
+ index_tensor = torch.tensor(m["indices"], dtype=torch.long)
+ values = torch.tensor(m["params"])
+ return index_tensor, values
+
+ def _pre_step(self):
+ """
+ Called at the beginning of step.
+
+ """
+ tensors_to_cat = [v.data.flatten() for _, v in self.model.state_dict().items()]
+ self.post_train_model = torch.cat(tensors_to_cat, dim=0)
+ self.g = self.e + (self.post_train_model - self.init_model)
+
+ def _averaging(self):
+ """
+ Averages the received model with the local model
+
+ """
+ with torch.no_grad():
+ total = torch.zeros_like(self.init_model)
+ weight_total = 0
+ for i, n in enumerate(self.caches):
+ if n != self.my_uid:
+ data = self.caches[n]
+ degree = len(self.graph.neighbors(n))
+ weight = 1 / (
+ max(len(self.peer_deques), degree) + 1
+ ) # Metro-Hastings
+ weight_total += weight
+ total += data * weight
+
+ total += (1 - weight_total) * self.caches[self.my_uid] # Metro-Hastings
+ self.avg = total
+
+ def step(self):
+ """
+ Perform a sharing step. Implements D-PSGD.
+
+ """
+
+ self._pre_step()
+
+ logging.info("Starting model averaging after receiving from all neighbors")
+ self._averaging()
+ logging.info("Model averaging complete")
+
+ self.u = (self.avg - self.init_model) - self.g
+
+ data = self.serialized_model()
+ my_uid = self.mapping.get_uid(self.rank, self.machine_id)
+ all_neighbors = self.graph.neighbors(my_uid)
+ iter_neighbors = self.get_neighbors(all_neighbors)
+ data["degree"] = len(all_neighbors)
+ data["iteration"] = self.communication_round
+ for neighbor in iter_neighbors:
+ self.communication.send(neighbor, data)
+
+ logging.info("Waiting for messages from neighbors")
+ while not self.received_from_all():
+ sender, data = self.communication.receive()
+ logging.debug("Received model from {}".format(sender))
+ degree = data["degree"]
+ iteration = data["iteration"]
+ del data["degree"]
+ del data["iteration"]
+ self.peer_deques[sender].append((degree, iteration, data))
+ logging.info(
+ "Deserialized received model from {} of iteration {}".format(
+ sender, iteration
+ )
+ )
+
+ for i, n in enumerate(self.peer_deques):
+ degree, iteration, data = self.peer_deques[n].popleft()
+ ind, val = self.deserialized_model(data)
+ self.caches[n][ind] += val
+
+ self.communication_round += 1
+
+ tensors_to_cat = [v.data.flatten() for _, v in self.model.state_dict().items()]
+ post_share_model = torch.cat(tensors_to_cat, dim=0)
+ self.init_model = post_share_model
+ self._post_step()
+
+ def save_vector(self, v, s):
+ """
+ Saves the given vector to the file.
+
+ Parameters
+ ----------
+ v : torch.tensor
+ The torch tensor to write to file
+ s : str
+ Path to folder to write to
+
+ """
+ output_dict = dict()
+ output_dict["order"] = list(self.model.state_dict().keys())
+ shapes = dict()
+ for k, v1 in self.model.state_dict().items():
+ shapes[k] = list(v1.shape)
+ output_dict["shapes"] = shapes
+
+ output_dict["tensor"] = v.tolist()
+
+ with open(
+ os.path.join(
+ s,
+ "{}.json".format(self.communication_round + 1),
+ ),
+ "w",
+ ) as of:
+ json.dump(output_dict, of)
+
+ def save_change(self):
+ """
+ Saves the change and the gradient values for every iteration
+
+ """
+ self.save_vector(self.model.model_change, self.model_change_path)
diff --git a/random files/WVFreq_Diff_Values.png b/random files/WVFreq_Diff_Values.png
new file mode 100644
index 0000000000000000000000000000000000000000..48954e2293cc2e105bb2eaa7c32a20df709bd91e
Binary files /dev/null and b/random files/WVFreq_Diff_Values.png differ
diff --git a/random files/WVFreq_Values.png b/random files/WVFreq_Values.png
new file mode 100644
index 0000000000000000000000000000000000000000..ee8f7ce2d6d0951098a75e86fcf01eb99d17f87d
Binary files /dev/null and b/random files/WVFreq_Values.png differ
diff --git a/random files/accHist.png b/random files/accHist.png
new file mode 100644
index 0000000000000000000000000000000000000000..6d14866f13987e00fa5de2ec071fd07e7462e59d
Binary files /dev/null and b/random files/accHist.png differ
diff --git a/random files/accPercentiles.png b/random files/accPercentiles.png
new file mode 100644
index 0000000000000000000000000000000000000000..71aea29b6fbadfc71eb99f4d48726be4ccdaba62
Binary files /dev/null and b/random files/accPercentiles.png differ
diff --git a/random files/accPercentiles70.png b/random files/accPercentiles70.png
new file mode 100644
index 0000000000000000000000000000000000000000..60175b0f7eff376e436ca78a314bd4e461fadaa9
Binary files /dev/null and b/random files/accPercentiles70.png differ
diff --git a/random files/celeba.ipynb b/random files/celeba.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..14aa8d39e80bf44968a2381323fe1be79209b238
--- /dev/null
+++ b/random files/celeba.ipynb
@@ -0,0 +1,768 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "ZMZYcW3itMzT",
+ "outputId": "f2970f7e-cf26-4a67-e8d3-29bcd1a11775"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "2VftlLfttdT8",
+ "outputId": "48b47fdc-853b-4711-ae95-8c0e64510615"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "ft7BMl1LyWP6"
+ },
+ "outputs": [],
+ "source": [
+ "from torch import nn\n",
+ "import torch\n",
+ "import os\n",
+ "import json\n",
+ "import pickle\n",
+ "import numpy as np\n",
+ "import pywt\n",
+ "train_dir = \"./../\"\n",
+ "my_train_data = {\"x\": [], \"y\": []}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<torch._C.Generator at 0x7f9ed4066cd0>"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.manual_seed(13)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "id": "hi0N5rB5xBWn"
+ },
+ "outputs": [],
+ "source": [
+ "if torch.cuda.is_available():\n",
+ " device = \"cuda\"\n",
+ "else:\n",
+ " device = \"cpu\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 35
+ },
+ "id": "6lO3uYsmxNYz",
+ "outputId": "b170b610-f21e-465d-fcd6-b7e6989e73e5"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'cpu'"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "device"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Contents\n",
+ "* [CNN Model Training](#train)\n",
+ "* [Optimizer analysis](#optim)\n",
+ "* [FFT](#fft)\n",
+ "* [Wavelets](#wt)\n",
+ "* [FFT Training](#ffttrain)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## CNN Model Training <a class=\"anchor\" id=\"train\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "id": "9LpgzEw1s-xo"
+ },
+ "outputs": [],
+ "source": [
+ "# From Femnist.py\n",
+ "def read_file(file_path):\n",
+ " with open(file_path, \"r\") as inf:\n",
+ " client_data = json.load(inf)\n",
+ " print(\"loaded the data\")\n",
+ " return (\n",
+ " client_data[\"users\"],\n",
+ " client_data[\"num_samples\"],\n",
+ " client_data[\"user_data\"],\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QBu1kiw8s-xr"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "jI3ixEN4s-xt",
+ "outputId": "ed969663-9e1e-4810-9507-52cdc426650a"
+ },
+ "source": [
+ "# From Femnist.py\n",
+ "for i in range(1):\n",
+ " cur_file = \"leaf/data/femnist/data/train/all_data_0_niid_0_keep_0_train_9.json\"\n",
+ " # test_file = \"leaf/data/femnist/data/test/all_data_0_niid_0_keep_0_test_9.json\"\n",
+ " # cur_file = test_file\n",
+ " clients, _, train_data = read_file(\n",
+ " os.path.join(train_dir, cur_file)\n",
+ " )\n",
+ " for cur_client in clients:\n",
+ " # self.clients.append(cur_client)\n",
+ " my_train_data[\"x\"].extend(train_data[cur_client][\"x\"])\n",
+ " my_train_data[\"y\"].extend(train_data[cur_client][\"y\"])\n",
+ " del train_data[cur_client]\n"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "wvHsSz8as-xw"
+ },
+ "source": [
+ "train_x = (\n",
+ " np.array(my_train_data[\"x\"], dtype=np.dtype(\"float32\"))\n",
+ " .reshape(-1, 28, 28, 1)\n",
+ " .transpose(0, 3, 1, 2)\n",
+ ")\n",
+ "train_y = np.array(my_train_data[\"y\"], dtype=np.dtype(\"int64\")).reshape(-1)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "K8X471SKs-xz",
+ "outputId": "cdf73c06-1323-4e76-850b-16324008d255"
+ },
+ "source": [
+ "len(train_y)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "EpWNELBrs-x0"
+ },
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"wb\") as f:\n",
+ " pickle.dump({\"test_x\": train_x, \"test_y\": train_y}, f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mAEASHr2s-x1"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "id": "Am_XlcSSs-x3"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "evAd9ZvYs-x6"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "id": "9_vIFakbs-x7",
+ "outputId": "3a8b546a-186f-4519-8c0b-e853986a8101"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(735856, 1, 28, 28)"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train[\"train_x\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "id": "GPyZ2C8ws-x9"
+ },
+ "outputs": [],
+ "source": [
+ "IMAGE_DIM = 84\n",
+ "CHANNELS = 3\n",
+ "NUM_CLASSES = 2\n",
+ "import torch.nn.functional as F\n",
+ "\n",
+ "class CNN(nn.Module):\n",
+ " \"\"\"\n",
+ " Class for a CNN Model for Celeba\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " def __init__(self):\n",
+ " \"\"\"\n",
+ " Constructor. Instantiates the CNN Model\n",
+ " with 84*84*3 Input and 2 output classes\n",
+ "\n",
+ " \"\"\"\n",
+ " super().__init__()\n",
+ " # 2.8k parameters\n",
+ " self.conv1 = nn.Conv2d(CHANNELS, 32, 3, padding=\"same\")\n",
+ " self.pool = nn.MaxPool2d(2, 2)\n",
+ " self.conv2 = nn.Conv2d(32, 32, 3, padding=\"same\")\n",
+ " self.conv3 = nn.Conv2d(32, 32, 3, padding=\"same\")\n",
+ " self.conv4 = nn.Conv2d(32, 32, 3, padding=\"same\")\n",
+ " self.fc1 = nn.Linear(5 * 5 * 32, NUM_CLASSES)\n",
+ "\n",
+ " def forward(self, x):\n",
+ " \"\"\"\n",
+ " Forward pass of the model\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " x : torch.tensor\n",
+ " The input torch tensor\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " torch.tensor\n",
+ " The output torch tensor\n",
+ "\n",
+ " \"\"\"\n",
+ " x = F.relu(self.pool(self.conv1(x)))\n",
+ " x = F.relu(self.pool(self.conv2(x)))\n",
+ " x = F.relu(self.pool(self.conv3(x)))\n",
+ " x = F.relu(self.pool(self.conv4(x)))\n",
+ " x = torch.flatten(x, 1)\n",
+ " x = self.fc1(x)\n",
+ " return x"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "bCgW8ClBs-x_"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "id": "oBGwcwZks-yA"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from torch.utils.data import Dataset\n",
+ "\n",
+ "class FemnistDataset(Dataset):\n",
+ " def __init__(self, training, transform=None, target_transform=None):\n",
+ " if training:\n",
+ " with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)\n",
+ " self.data = train[\"train_x\"]\n",
+ " self.label = train[\"train_y\"]\n",
+ " else: \n",
+ " with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)\n",
+ " self.data = test[\"test_x\"]\n",
+ " self.label = test[\"test_y\"]\n",
+ " self.transform = transform\n",
+ " self.target_transform = target_transform\n",
+ "\n",
+ " def __len__(self):\n",
+ " return len(self.label)\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " return self.data[idx], self.label[idx]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "id": "U3boC_N4s-yC"
+ },
+ "outputs": [],
+ "source": [
+ "from torch.utils.data import DataLoader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "id": "sJsrQXkEs-yD"
+ },
+ "outputs": [],
+ "source": [
+ "trainset = FemnistDataset(True)\n",
+ "testset = FemnistDataset(False)\n",
+ "\n",
+ "train_dataloader = DataLoader(trainset, batch_size=128, shuffle=True)\n",
+ "test_dataloader = DataLoader(testset, batch_size=128, shuffle=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "5749"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(train_dataloader)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "735872"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "5749*128"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {
+ "id": "e65Izyv0s-yE"
+ },
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1714, -0.1405, -0.0313, 0.0578, 0.0766, -0.1935, 0.1538, 0.0907,\n",
+ " 0.0226, 0.0708, 0.1525, -0.0412, -0.1595, 0.0278, 0.0404, -0.1018,\n",
+ " -0.1330, -0.1515, -0.1124, 0.0592, -0.0866, -0.0707, -0.0435, -0.0559,\n",
+ " 0.0480, -0.1483, -0.0215, -0.1404, -0.0433, 0.0631, -0.1822, -0.0755])"
+ ]
+ },
+ "execution_count": 28,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(model.state_dict().values())[1]"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "tensor([-0.0180, 0.0236, 0.1279, -0.1352, -0.1948, -0.0330, -0.1615, -0.0286,\n",
+ " -0.1762, 0.0040, 0.1570, -0.1069, -0.1074, -0.1417, -0.1171, 0.0359,\n",
+ " 0.1276, -0.1534, -0.1773, -0.1639, 0.1334, 0.0518, 0.0586, 0.1466,\n",
+ " 0.1283, 0.0443, -0.0982, -0.1739, -0.0061, 0.1047, -0.0291, 0.1525])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "torch.Size([32, 1, 5, 5])\n",
+ "torch.Size([32])\n",
+ "torch.Size([64, 32, 5, 5])\n",
+ "torch.Size([64])\n",
+ "torch.Size([512, 3136])\n",
+ "torch.Size([512])\n",
+ "torch.Size([62, 512])\n",
+ "torch.Size([62])\n"
+ ]
+ }
+ ],
+ "source": [
+ "for p in model.parameters():\n",
+ " print(p.data.size())\n",
+ " p.data = torch.zeros(p.data.size())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
+ " 0., 0., 0., 0., 0., 0., 0., 0.])"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(model.state_dict().values())[1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "eqOXilqMs-yF",
+ "outputId": "06799a3b-983b-4f51-a7bd-a901c041bd05"
+ },
+ "outputs": [],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(10):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {
+ "id": "4P-VA0vcs-yH"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"/results:128:\"+str(lr)+\".pkl\", \"wb\") as f:\n",
+ " pickle.dump(stats, f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "641-b_VCvT2b",
+ "outputId": "cced38ab-5c04-45b2-faf4-e73327126159"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 60,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "F_OKqiiHs-yJ",
+ "outputId": "65786b88-05f4-42fa-a851-03397ef4457a"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.1: [9. 3.69780584 5.50521373]\n",
+ "0.01: [ 9. 3.98475619 82.61967193]\n",
+ "0.005: [ 9. 0.51492128 85.40642722]\n",
+ "0.001: [ 9. 0.41047618 88.03829502]\n",
+ "0.0005: [ 9. 0.44351858 88.21025672]\n",
+ "0.0001: [ 9. 0.67233266 87.71754375]\n",
+ "1e-05: [ 9. 1.81167539 81.52570279]\n"
+ ]
+ }
+ ],
+ "source": [
+ "lrs = [0.1, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00001]\n",
+ "for l in lrs:\n",
+ " with open(train_dir+\"/results:128:\"+str(l)+\".pkl\", \"rb\") as f:\n",
+ " res = pickle.load(f)\n",
+ " print(str(l)+\": \" + str(np.amax(res[\"test\"], axis=0)))#+ str(np.max(res[\"test\"]))\n",
+ " # print(str(l)+\": \" + str(res[\"test\"]))\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "rADw-XkfKjOo",
+ "outputId": "06c54a2c-f7c2-4610-f879-3e1c2f98543f"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.1: [[0, 3.6898819485246297, 4.914933837429111], [1, 3.695287103771977, 4.914933837429111], [2, 3.691172517592003, 5.505213732544667], [3, 3.6920483804158226, 4.967376059515824], [4, 3.6939517986755845, 5.505213732544667], [5, 3.6917366434742993, 4.914933837429111], [6, 3.695435837910811, 5.505213732544667], [7, 3.6978058357506574, 5.135679004817367], [8, 3.6948341036363623, 5.505213732544667], [9, 3.6921330658768343, 5.135679004817367]]\n",
+ "0.01: [[0, 0.5728003431500958, 81.8086468687115], [1, 0.5517885946725348, 82.61967193121532], [2, 3.984756194857093, 25.844258796268065], [3, 0.5739870879932797, 81.7476675407037], [4, 0.7832032613188912, 75.77779132873955], [5, 0.7142617320772638, 77.80474419171901], [6, 0.6602287095348103, 79.28654186230868], [7, 0.6738644539380036, 79.2719068235868], [8, 0.6469118589079138, 79.77071772669065], [9, 0.6788249858734946, 79.28898103542899]]\n",
+ "0.005: [[0, 0.4834194714537649, 83.79047502896519], [1, 0.466142692822562, 84.33928898103544], [2, 0.4559767278791776, 84.9515214342338], [3, 0.4488265364432298, 84.86493078846271], [4, 0.4554814101660307, 84.773461796451], [5, 0.5149212768315897, 83.05872309287152], [6, 0.4551808235472338, 84.86127202878224], [7, 0.4531376465992325, 85.06494298432831], [8, 0.4589428385362238, 84.83078236477834], [9, 0.4409179601951992, 85.40642722117202]]\n",
+ "0.001: [[0, 0.4104761779773254, 85.89670101835478], [1, 0.36889259526491536, 87.17604731995854], [2, 0.3517718464717292, 87.6992499542655], [3, 0.35526543692939927, 87.57607171168974], [4, 0.3493265717198808, 87.76266845539362], [5, 0.35079776836259874, 87.47362644063662], [6, 0.34534544340812845, 87.96268065125923], [7, 0.35734797465540874, 87.72608085858894], [8, 0.3524193228360457, 87.63339228001708], [9, 0.35447056082407136, 88.0382950179889]]\n",
+ "0.0005: [[0, 0.4435185831906085, 85.1039697542533], [1, 0.37539843543085405, 86.94310628696871], [2, 0.35873422210283473, 87.3797182755046], [3, 0.34818319706667605, 87.93097140069517], [4, 0.34545205666010914, 87.86633331300689], [5, 0.3371337376732536, 88.10415269223734], [6, 0.33852135716659976, 88.11512897127874], [7, 0.33852605533302293, 88.14074028904201], [8, 0.33997187332225476, 88.21025672297091], [9, 0.3402654077747311, 88.1968412708092]]\n",
+ "0.0001: [[0, 0.6723326555745278, 79.72437343740472], [1, 0.5084800024207409, 83.89901823281907], [2, 0.45863669222676995, 84.88932251966584], [3, 0.42524330169194946, 85.90767729739618], [4, 0.4028480564841242, 86.33575218001097], [5, 0.38621816764383715, 86.9126166229648], [6, 0.3782781209337544, 87.11506799195074], [7, 0.3759017101781045, 87.00530520153667], [8, 0.3668581307538772, 87.32117812061712], [9, 0.3569657983208967, 87.71754375266785]]\n",
+ "1e-05: [[0, 1.8116753936371826, 54.021586682114766], [1, 1.3575628893609724, 63.8148667601683], [2, 1.0996285610935432, 69.48716385145435], [3, 0.9349309672803477, 73.7788889566437], [4, 0.8294046315685635, 76.08878590157937], [5, 0.7614829346374863, 77.91938532837368], [6, 0.7032811826737176, 79.30727483383133], [7, 0.6657257149818349, 80.09024940545156], [8, 0.6363296165202226, 80.82931886090616], [9, 0.6094131586890139, 81.52570278675529]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "lrs = [0.1, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00001]\n",
+ "for l in lrs:\n",
+ " with open(train_dir+\"/results:128:\"+str(l)+\".pkl\", \"rb\") as f:\n",
+ " res = pickle.load(f)\n",
+ " # print(str(l)+\": \" + str(np.amax(res[\"test\"], axis=0)))#+ str(np.max(res[\"test\"]))\n",
+ " print(str(l)+\": \" + str(res[\"test\"]))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "HGpNYzG_s-yJ",
+ "outputId": "783622a5-249f-4dd8-d242-fc6dfa47443c"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using cache found in /home/jeffrey/.cache/torch/hub/pytorch_vision_v0.10.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "import torch\n",
+ "resnet = torch.hub.load('pytorch/vision:v0.10.0', 'resnet18', pretrained=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "uZFgT6wss-yL",
+ "outputId": "10f8fc51-abb7-4c2b-f608-85229f3de29d",
+ "tags": []
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "11699132\n"
+ ]
+ }
+ ],
+ "source": [
+ "total = 0\n",
+ "for i in resnet.state_dict().values():\n",
+ " total += i.flatten().size(dim=0)\n",
+ "print(total)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Optimizer analysis <a class=\"anchor\" id=\"optim\"></a>"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "name": "learningrate.ipynb",
+ "provenance": []
+ },
+ "interpreter": {
+ "hash": "996934296aa9d79be6c3d800a38d8fdb7dfa8fe7bb07df178f1397cde2cb8742"
+ },
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/random files/generate_graph.py b/random files/generate_graph.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd625424844ad9d4d95aa885da311bd6bc76dbfb
--- /dev/null
+++ b/random files/generate_graph.py
@@ -0,0 +1,19 @@
+import matplotlib.pyplot as plt
+import networkx as nx
+
+from decentralizepy.graphs.Regular import Regular
+from decentralizepy.graphs.Ring import Ring
+from decentralizepy.graphs.Star import Star
+
+# b = Regular(16, 1, 686)
+
+
+b = Regular(96*3, 5)
+# TODO: rewrite to directly connect dissconnected subgraphs
+# b.connect_graph()
+
+b.write_graph_to_file(f"{96*3}_regular.edges")
+
+g = nx.read_edgelist(f"{96*3}_regular.edges")
+nx.draw(g)
+#plt.savefig("96_star.png")
diff --git a/random files/ip_addr_1Machines.json b/random files/ip_addr_1Machines.json
new file mode 100644
index 0000000000000000000000000000000000000000..15d6591df53574707ac03627fa19c9ecd749b1e3
--- /dev/null
+++ b/random files/ip_addr_1Machines.json
@@ -0,0 +1,3 @@
+{
+ "0": "127.0.0.1"
+}
\ No newline at end of file
diff --git a/random files/learningrate-Copy2.ipynb b/random files/learningrate-Copy2.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..6a5dff362073514bfe9ce3e1c18bad8f0a11bb17
--- /dev/null
+++ b/random files/learningrate-Copy2.ipynb
@@ -0,0 +1,7121 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "ZMZYcW3itMzT",
+ "outputId": "f2970f7e-cf26-4a67-e8d3-29bcd1a11775"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "2VftlLfttdT8",
+ "outputId": "48b47fdc-853b-4711-ae95-8c0e64510615"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "ft7BMl1LyWP6"
+ },
+ "outputs": [],
+ "source": [
+ "from torch import nn\n",
+ "import torch\n",
+ "import os\n",
+ "import json\n",
+ "import pickle\n",
+ "import numpy as np\n",
+ "import pywt\n",
+ "#from torchvision.models.resnet import ResNet, BasicBlock, Bottleneck\n",
+ "train_dir = \"../../\"\n",
+ "my_train_data = {\"x\": [], \"y\": []}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<torch._C.Generator at 0x7f1e10694c30>"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.manual_seed(13)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "id": "hi0N5rB5xBWn"
+ },
+ "outputs": [],
+ "source": [
+ "if torch.cuda.is_available():\n",
+ " device = \"cuda\"\n",
+ "else:\n",
+ " device = \"cpu\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 35
+ },
+ "id": "6lO3uYsmxNYz",
+ "outputId": "b170b610-f21e-465d-fcd6-b7e6989e73e5"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'cpu'"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "device"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "torch.set_num_threads(6)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Contents\n",
+ "* [CNN Model Training](#train)\n",
+ "* [Optimizer analysis](#optim)\n",
+ "* [FFT](#fft)\n",
+ "* [Wavelets](#wt)\n",
+ "* [FFT Training](#ffttrain)\n",
+ "* [Node_Training](#nodetraining)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## CNN Model Training <a class=\"anchor\" id=\"train\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "id": "9LpgzEw1s-xo"
+ },
+ "outputs": [],
+ "source": [
+ "# From Femnist.py\n",
+ "def read_file(file_path):\n",
+ " with open(file_path, \"r\") as inf:\n",
+ " client_data = json.load(inf)\n",
+ " print(\"loaded the data\")\n",
+ " return (\n",
+ " client_data[\"users\"],\n",
+ " client_data[\"num_samples\"],\n",
+ " client_data[\"user_data\"],\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QBu1kiw8s-xr"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "jI3ixEN4s-xt",
+ "outputId": "ed969663-9e1e-4810-9507-52cdc426650a"
+ },
+ "source": [
+ "# From Femnist.py\n",
+ "for i in range(1):\n",
+ " cur_file = \"leaf/data/femnist/data/train/all_data_0_niid_0_keep_0_train_9.json\"\n",
+ " # test_file = \"leaf/data/femnist/data/test/all_data_0_niid_0_keep_0_test_9.json\"\n",
+ " # cur_file = test_file\n",
+ " clients, _, train_data = read_file(\n",
+ " os.path.join(train_dir, cur_file)\n",
+ " )\n",
+ " for cur_client in clients:\n",
+ " # self.clients.append(cur_client)\n",
+ " my_train_data[\"x\"].extend(train_data[cur_client][\"x\"])\n",
+ " my_train_data[\"y\"].extend(train_data[cur_client][\"y\"])\n",
+ " del train_data[cur_client]\n"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "wvHsSz8as-xw"
+ },
+ "source": [
+ "train_x = (\n",
+ " np.array(my_train_data[\"x\"], dtype=np.dtype(\"float32\"))\n",
+ " .reshape(-1, 28, 28, 1)\n",
+ " .transpose(0, 3, 1, 2)\n",
+ ")\n",
+ "train_y = np.array(my_train_data[\"y\"], dtype=np.dtype(\"int64\")).reshape(-1)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "K8X471SKs-xz",
+ "outputId": "cdf73c06-1323-4e76-850b-16324008d255"
+ },
+ "source": [
+ "len(train_y)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "EpWNELBrs-x0"
+ },
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"wb\") as f:\n",
+ " pickle.dump({\"test_x\": train_x, \"test_y\": train_y}, f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mAEASHr2s-x1"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "Am_XlcSSs-x3"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "id": "evAd9ZvYs-x6"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "id": "9_vIFakbs-x7",
+ "outputId": "3a8b546a-186f-4519-8c0b-e853986a8101"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(735856, 1, 28, 28)"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train[\"train_x\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):\n",
+ " \"\"\"3x3 convolution with padding\"\"\"\n",
+ " return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
+ " padding=dilation, groups=groups, bias=False, dilation=dilation)\n",
+ "\n",
+ "\n",
+ "def conv1x1(in_planes, out_planes, stride=1):\n",
+ " \"\"\"1x1 convolution\"\"\"\n",
+ " return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n",
+ "\n",
+ "class BasicBlock(nn.Module):\n",
+ " expansion = 1\n",
+ "\n",
+ " def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n",
+ " base_width=64, dilation=1, norm_layer=None):\n",
+ " super(BasicBlock, self).__init__()\n",
+ " if norm_layer is None:\n",
+ " norm_layer = nn.BatchNorm2d\n",
+ " if dilation > 1:\n",
+ " raise NotImplementedError(\"Dilation > 1 not supported in BasicBlock\")\n",
+ " # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n",
+ " self.conv1 = conv3x3(inplanes, planes, stride)\n",
+ " self.bn1 = norm_layer(planes)\n",
+ " self.relu = nn.ReLU(inplace=True)\n",
+ " self.conv2 = conv3x3(planes, planes)\n",
+ " self.bn2 = norm_layer(planes)\n",
+ " self.downsample = downsample\n",
+ " self.stride = stride\n",
+ " \n",
+ " def forward(self, x):\n",
+ " identity = x\n",
+ "\n",
+ " out = self.conv1(x)\n",
+ " out = self.bn1(out)\n",
+ " out = self.relu(out)\n",
+ "\n",
+ " out = self.conv2(out)\n",
+ " out = self.bn2(out)\n",
+ "\n",
+ " if self.downsample is not None:\n",
+ " identity = self.downsample(x)\n",
+ "\n",
+ " out += identity\n",
+ " out = self.relu(out)\n",
+ "\n",
+ " return out\n",
+ "\n",
+ "class Bottleneck(nn.Module):\n",
+ " # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)\n",
+ " # while original implementation places the stride at the first 1x1 convolution(self.conv1)\n",
+ " # according to \"Deep residual learning for image recognition\"https://arxiv.org/abs/1512.03385.\n",
+ " # This variant is also known as ResNet V1.5 and improves accuracy according to\n",
+ " # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.\n",
+ "\n",
+ " expansion = 4\n",
+ "\n",
+ " def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n",
+ " base_width=64, dilation=1, norm_layer=None):\n",
+ " super(Bottleneck, self).__init__()\n",
+ " if norm_layer is None:\n",
+ " norm_layer = nn.BatchNorm2d\n",
+ " width = int(planes * (base_width / 64.)) * groups\n",
+ " # Both self.conv2 and self.downsample layers downsample the input when stride != 1\n",
+ " self.conv1 = conv1x1(inplanes, width)\n",
+ " self.bn1 = norm_layer(width)\n",
+ " self.conv2 = conv3x3(width, width, stride, groups, dilation)\n",
+ " self.bn2 = norm_layer(width)\n",
+ " self.conv3 = conv1x1(width, planes * self.expansion)\n",
+ " self.bn3 = norm_layer(planes * self.expansion)\n",
+ " self.relu = nn.ReLU(inplace=True)\n",
+ " self.downsample = downsample\n",
+ " self.stride = stride\n",
+ "\n",
+ " def forward(self, x):\n",
+ " identity = x\n",
+ "\n",
+ " out = self.conv1(x)\n",
+ " out = self.bn1(out)\n",
+ " out = self.relu(out)\n",
+ "\n",
+ " out = self.conv2(out)\n",
+ " out = self.bn2(out)\n",
+ " out = self.relu(out)\n",
+ "\n",
+ " out = self.conv3(out)\n",
+ " out = self.bn3(out)\n",
+ "\n",
+ " if self.downsample is not None:\n",
+ " identity = self.downsample(x)\n",
+ "\n",
+ " out += identity\n",
+ " out = self.relu(out)\n",
+ "\n",
+ " return out\n",
+ "\n",
+ "class ResNet(nn.Module):\n",
+ "\n",
+ " def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,\n",
+ " groups=1, width_per_group=64, replace_stride_with_dilation=None,\n",
+ " norm_layer=None):\n",
+ " super(ResNet, self).__init__()\n",
+ " if norm_layer is None:\n",
+ " norm_layer = nn.BatchNorm2d\n",
+ " self._norm_layer = norm_layer\n",
+ "\n",
+ " self.inplanes = 32\n",
+ " self.dilation = 1\n",
+ " if replace_stride_with_dilation is None:\n",
+ " # each element in the tuple indicates if we should replace\n",
+ " # the 2x2 stride with a dilated convolution instead\n",
+ " replace_stride_with_dilation = [False, False, False]\n",
+ " if len(replace_stride_with_dilation) != 3:\n",
+ " raise ValueError(\"replace_stride_with_dilation should be None \"\n",
+ " \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n",
+ " self.groups = groups\n",
+ " self.base_width = width_per_group\n",
+ " self.conv1 = nn.Conv2d(1, self.inplanes, kernel_size=7, stride=2, padding=3,\n",
+ " bias=False)\n",
+ " self.bn1 = norm_layer(self.inplanes)\n",
+ " self.relu = nn.ReLU(inplace=True)\n",
+ " self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
+ " self.layer1 = self._make_layer(block, 32, layers[0])\n",
+ " self.layer2 = self._make_layer(block, 64, layers[1], stride=2,\n",
+ " dilate=replace_stride_with_dilation[0])\n",
+ " self.layer3 = self._make_layer(block, 128, layers[2], stride=2,\n",
+ " dilate=replace_stride_with_dilation[1])\n",
+ " self.layer4 = self._make_layer(block, 256, layers[3], stride=2,\n",
+ " dilate=replace_stride_with_dilation[2])\n",
+ " self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
+ " self.fc = nn.Linear(256 * block.expansion, num_classes)\n",
+ "\n",
+ " for m in self.modules():\n",
+ " if isinstance(m, nn.Conv2d):\n",
+ " nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n",
+ " elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n",
+ " nn.init.constant_(m.weight, 1)\n",
+ " nn.init.constant_(m.bias, 0)\n",
+ "\n",
+ " # Zero-initialize the last BN in each residual branch,\n",
+ " # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n",
+ " # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n",
+ " if zero_init_residual:\n",
+ " for m in self.modules():\n",
+ " if isinstance(m, Bottleneck):\n",
+ " nn.init.constant_(m.bn3.weight, 0)\n",
+ " elif isinstance(m, BasicBlock):\n",
+ " nn.init.constant_(m.bn2.weight, 0)\n",
+ "\n",
+ " def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n",
+ " norm_layer = self._norm_layer\n",
+ " downsample = None\n",
+ " previous_dilation = self.dilation\n",
+ " if dilate:\n",
+ " self.dilation *= stride\n",
+ " stride = 1\n",
+ " if stride != 1 or self.inplanes != planes * block.expansion:\n",
+ " downsample = nn.Sequential(\n",
+ " conv1x1(self.inplanes, planes * block.expansion, stride),\n",
+ " norm_layer(planes * block.expansion),\n",
+ " )\n",
+ "\n",
+ " layers = []\n",
+ " layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n",
+ " self.base_width, previous_dilation, norm_layer))\n",
+ " self.inplanes = planes * block.expansion\n",
+ " for _ in range(1, blocks):\n",
+ " layers.append(block(self.inplanes, planes, groups=self.groups,\n",
+ " base_width=self.base_width, dilation=self.dilation,\n",
+ " norm_layer=norm_layer))\n",
+ "\n",
+ " return nn.Sequential(*layers)\n",
+ "\n",
+ " def _forward_impl(self, x):\n",
+ " # See note [TorchScript super()]\n",
+ " x = self.conv1(x)\n",
+ " x = self.bn1(x)\n",
+ " x = self.relu(x)\n",
+ " x = self.maxpool(x)\n",
+ "\n",
+ " x = self.layer1(x)\n",
+ " x = self.layer2(x)\n",
+ " x = self.layer3(x)\n",
+ " x = self.layer4(x)\n",
+ "\n",
+ " x = self.avgpool(x)\n",
+ " x = torch.flatten(x, 1)\n",
+ " x = self.fc(x)\n",
+ "\n",
+ " return x\n",
+ "\n",
+ " def forward(self, x):\n",
+ " return self._forward_impl(x)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "id": "GPyZ2C8ws-x9"
+ },
+ "outputs": [],
+ "source": [
+ "NUM_CLASSES = 62\n",
+ "IMAGE_SIZE = (28, 28)\n",
+ "FLAT_SIZE = 28 * 28\n",
+ "PIXEL_RANGE = 256.0\n",
+ "\n",
+ "model = ResNet(BasicBlock, [2,2,2,2], num_classes=62, width_per_group=32)\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "id": "bCgW8ClBs-x_"
+ },
+ "outputs": [],
+ "source": [
+ "flat = []\n",
+ "for v in model.state_dict().values():\n",
+ " flat.append(v.flatten())\n",
+ "conc = torch.cat(flat)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "torch.Size([2816498])"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "conc.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "id": "oBGwcwZks-yA"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from torch.utils.data import Dataset\n",
+ "\n",
+ "class FemnistDataset(Dataset):\n",
+ " def __init__(self, training, transform=None, target_transform=None):\n",
+ " if training:\n",
+ " with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)\n",
+ " self.data = train[\"train_x\"]\n",
+ " self.label = train[\"train_y\"]\n",
+ " else: \n",
+ " with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)\n",
+ " self.data = test[\"test_x\"]\n",
+ " self.label = test[\"test_y\"]\n",
+ " self.transform = transform\n",
+ " self.target_transform = target_transform\n",
+ "\n",
+ " def __len__(self):\n",
+ " return len(self.label)\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " return self.data[idx], self.label[idx]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "id": "U3boC_N4s-yC"
+ },
+ "outputs": [],
+ "source": [
+ "from torch.utils.data import DataLoader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {
+ "id": "sJsrQXkEs-yD"
+ },
+ "outputs": [],
+ "source": [
+ "trainset = FemnistDataset(True)\n",
+ "testset = FemnistDataset(False)\n",
+ "\n",
+ "train_dataloader = DataLoader(trainset, batch_size=128, shuffle=True)\n",
+ "test_dataloader = DataLoader(testset, batch_size=128, shuffle=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "id": "e65Izyv0s-yE"
+ },
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = model.to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
+ " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(model.state_dict().values())[1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for v in model.state_dict().values():\n",
+ " flat.append(v.flatten())\n",
+ "conc = torch.cat(flat)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor([-3.6153e-02, 1.0922e-02, 1.5911e-01, 7.6926e-02, 5.7525e-02,\n",
+ " 2.0833e-01, 4.8255e-02, 2.0805e-01, -2.4887e-02, 8.3241e-02,\n",
+ " 5.7281e-02, 3.2598e-02, -5.9614e-03, 2.3448e-01, -2.7755e-02,\n",
+ " 1.2072e-01, 2.0834e-03, 1.4729e-01, -2.0169e-01, -2.3126e-03,\n",
+ " -1.0632e-02, 1.1928e-02, -1.2216e-01, -1.3769e-03, -1.1989e-01,\n",
+ " -2.1655e-02, 1.6833e-02, -1.0780e-01, 5.3008e-02, 1.1340e-01,\n",
+ " -1.1262e-01, -2.9909e-02, -1.3595e-01, 1.1996e-01, -3.5497e-02,\n",
+ " -5.9646e-02, -8.5283e-03, -3.5111e-02, -7.9876e-03, 4.3423e-02,\n",
+ " 8.0446e-02, 1.3475e-01, -1.7352e-02, 1.5214e-01, 5.7564e-02,\n",
+ " -1.2092e-02, 4.2873e-02, -4.6847e-02, -1.5831e-01, 3.2632e-02,\n",
+ " 1.2103e-02, -7.3686e-02, 5.8829e-02, -4.9315e-02, 1.1395e-02,\n",
+ " -1.2393e-02, 3.4627e-02, -2.0724e-02, -7.6100e-02, 1.5033e-02,\n",
+ " -6.2240e-02, 1.3045e-01, 2.9429e-02, 5.1437e-02, 5.2329e-02,\n",
+ " 2.4896e-02, -1.8821e-02, -4.8809e-02, -6.7213e-02, -1.0350e-02,\n",
+ " -9.7824e-03, 8.3952e-02, 7.2283e-02, -6.4382e-02, 1.5534e-01,\n",
+ " 8.4570e-02, -1.6595e-01, 4.1408e-03, 4.5516e-02, -5.6906e-02,\n",
+ " -4.2940e-02, -1.1772e-02, -5.6404e-02, 1.8210e-01, 2.0227e-02,\n",
+ " 7.7609e-02, -1.4693e-01, -6.8822e-02, -3.4503e-02, 4.1717e-02,\n",
+ " 6.6086e-02, 1.1133e-01, 1.3154e-02, -3.4234e-02, 9.1784e-04,\n",
+ " -2.9296e-02, 2.6286e-02, -1.1241e-01, -8.3519e-02, 1.9695e-01,\n",
+ " -9.7168e-02, -5.8979e-02, 1.5797e-02, 3.9749e-02, 3.8752e-02,\n",
+ " -7.1771e-02, -7.8542e-02, -1.0243e-02, 1.7219e-01, -1.1095e-01,\n",
+ " 1.2735e-01, -4.5902e-02, -9.4965e-03, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
+ " 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00, 1.0000e+00,\n",
+ " 1.0000e+00, 0.0000e+00, 1.6987e-01, -1.2518e-02, 7.4222e-02,\n",
+ " 8.1779e-04, 1.0769e-01, 1.7932e-01, 1.0626e-01, 5.5261e-02,\n",
+ " -7.1285e-02, 3.4232e-02, -1.9942e-01, -3.2020e-02, -5.8090e-04,\n",
+ " -3.1324e-03, -5.2624e-02, -1.1991e-01, -6.3235e-02, -5.6224e-02,\n",
+ " -3.8221e-03, 1.7819e-02, 1.5929e-01, -1.2379e-01, -1.1942e-01,\n",
+ " -9.9741e-02, 3.7306e-02, -4.5313e-02, 9.5044e-02, -1.0798e-01,\n",
+ " 4.7692e-02, -6.8353e-02, -6.4735e-02, 1.6302e-01, -7.3956e-02,\n",
+ " -5.7375e-02, -7.0186e-02, -1.9878e-02, 6.8199e-02, -7.7473e-02,\n",
+ " 1.1649e-01, -1.7948e-01, 6.5320e-02, -1.6462e-01, -1.7468e-01,\n",
+ " 6.9575e-02, -3.6026e-02, -1.8932e-02, 3.2033e-02, 3.8191e-02,\n",
+ " 4.2004e-02, 9.8125e-03, -2.4546e-02, 8.1653e-03, -4.7182e-02,\n",
+ " -1.4758e-01, 3.9359e-02, -2.0074e-02, 5.3813e-02, 1.3991e-02,\n",
+ " -3.9933e-02, -9.2483e-02, -6.1893e-02, 2.6557e-03, 1.9068e-02,\n",
+ " 4.8750e-02, 6.6708e-02, 1.9469e-02, -2.1096e-02, 4.4306e-02,\n",
+ " -9.8441e-02, 3.5618e-03, -7.8726e-02, 9.7077e-02, -2.4202e-02,\n",
+ " 1.7504e-02, 1.4211e-01, 6.3227e-02, 2.9580e-03, -2.3347e-02,\n",
+ " 5.8336e-02, -5.9110e-02, 7.2662e-02, 5.8302e-02, -6.3689e-02,\n",
+ " -1.7816e-02, -1.3492e-01, 1.1300e-01, -3.2642e-02, -4.5522e-02,\n",
+ " -8.7756e-02, 2.1591e-02, 1.7699e-01, 1.1176e-02, -4.9630e-02,\n",
+ " -1.1987e-01, 8.9175e-02, -1.5066e-02, -1.0060e-02, -3.6194e-03,\n",
+ " 6.5334e-02, 7.3754e-02, 1.2938e-02, -3.8157e-02, 4.3455e-02,\n",
+ " 2.0767e-02, 6.2418e-02, -3.1317e-03, -6.5233e-03, -2.9303e-02,\n",
+ " 5.8986e-02, 3.9827e-02, -4.4528e-02, 3.3096e-02, 1.4558e-01,\n",
+ " -7.1790e-02, 2.8021e-02, -1.1121e-01, -8.7729e-02, -1.6679e-02,\n",
+ " 6.0385e-02, -1.0069e-01, 3.8162e-03, -1.5804e-02, 9.3877e-03,\n",
+ " -1.9549e-02, 6.8907e-02, 9.2226e-02, 6.7532e-02, 5.1782e-02,\n",
+ " -5.0173e-02, 6.4691e-02, 5.3740e-03, 2.6782e-02, 1.7140e-01,\n",
+ " -7.1114e-02, 1.3247e-01, 1.9341e-02, -2.8109e-02, -9.6079e-02,\n",
+ " -1.6598e-01, 1.1776e-01, -1.2546e-01, 8.9671e-02, -1.1108e-01,\n",
+ " 4.8116e-03, 1.5545e-02, 1.0199e-01, -9.5309e-02, -1.4364e-03,\n",
+ " -4.8351e-02, -8.6616e-02, 2.2615e-02, -5.3860e-02, -4.1553e-02,\n",
+ " -3.7117e-02, -1.3335e-01, 6.1934e-02, -6.1574e-03, -4.8353e-04,\n",
+ " -6.7036e-02, 9.5313e-03, 6.4967e-02, 1.0548e-01, -8.5173e-02,\n",
+ " 1.6937e-02, 1.8506e-01, 1.1589e-01, 1.5855e-02, 5.2601e-02,\n",
+ " -1.9648e-02, -2.8683e-02, 1.7219e-02, 4.4335e-02, -1.2565e-01,\n",
+ " -1.1790e-01, 2.4435e-04, 1.0657e-01, 4.9675e-02, -1.5611e-02,\n",
+ " -1.0195e-02, -2.6327e-02, 6.7269e-02, -2.0982e-02, 1.9502e-03,\n",
+ " -1.5793e-02, 2.6358e-02, 6.0283e-02, 1.3618e-01, 1.3374e-01,\n",
+ " -2.5466e-02, -4.8720e-02, -1.8593e-01, 1.7561e-03, -1.2250e-01,\n",
+ " -1.6137e-01, -5.0002e-03, -1.2871e-01, -3.7437e-02, -4.0313e-02,\n",
+ " 3.4365e-02, -3.7071e-02, 2.2959e-02, 1.1128e-01, -4.9545e-03,\n",
+ " -2.6472e-02, 3.2626e-02, -1.7230e-02, 7.8508e-02, 2.7197e-02,\n",
+ " -9.8706e-02, 1.1325e-02, -4.3507e-02, 3.5809e-02, -8.2735e-02,\n",
+ " -3.1467e-02, -2.3812e-03, 2.2783e-01, 1.4469e-01, 4.7378e-02,\n",
+ " 1.1900e-01, 1.3966e-02, -1.5898e-01, -5.6670e-02, 6.0425e-02,\n",
+ " 4.4461e-02, 4.3565e-02, -1.3006e-01, -3.5693e-02, -1.6811e-02,\n",
+ " 7.3669e-02, -5.5727e-03, 2.3922e-02, -1.0258e-01, -2.0600e-02,\n",
+ " 7.0644e-02, 5.4080e-02, -1.8225e-02, 8.2288e-02, 1.3712e-01,\n",
+ " 9.1076e-03, -5.9369e-02, -1.2339e-01, -9.4833e-02, 3.3790e-02,\n",
+ " 2.7712e-02, 1.1237e-01, 9.4319e-02, 6.9273e-03, 2.7316e-02,\n",
+ " -2.6998e-02, 6.5125e-02, 3.9116e-02, -1.7837e-02, -7.1617e-02,\n",
+ " 4.7388e-02, 6.2827e-02, 8.7470e-02, -1.4028e-02, 3.4789e-02,\n",
+ " -1.0830e-01, 2.5725e-03, 7.5497e-02, -5.1828e-02, 2.0584e-01,\n",
+ " -2.0988e-02, 1.3311e-01, 1.8669e-02, -1.0620e-01, 6.3515e-02,\n",
+ " 2.3155e-02, 5.2781e-02, 9.0113e-02, -1.6853e-01, 3.3725e-02,\n",
+ " -8.8361e-02, 9.8612e-02, -7.9768e-02, 2.4839e-02, 5.0955e-02,\n",
+ " 1.9578e-01, 4.9857e-02, -4.9117e-02, -6.0971e-02, -7.1335e-02,\n",
+ " 4.8186e-03, 2.6048e-01, -4.1628e-02, 6.4097e-02, -6.2915e-02,\n",
+ " -2.2990e-02, -1.0170e-01, 7.9752e-02, 1.4351e-01, -5.9676e-02,\n",
+ " 1.0420e-02, -2.1729e-02, 3.7526e-02, -3.2499e-02, 7.3554e-02,\n",
+ " -1.1204e-01, -7.0101e-02, -1.3232e-01, -8.4415e-02, -7.2395e-02,\n",
+ " 1.0182e-01, -3.3702e-03, 5.1951e-02, 5.0360e-02, -7.8748e-02,\n",
+ " 7.5663e-02, 8.7619e-02, 5.2736e-02, 1.0218e-01, 1.5727e-02,\n",
+ " -3.0746e-02, 5.0295e-02, 5.2703e-03, 5.5152e-02, -9.1586e-02,\n",
+ " -2.4175e-02, -8.0762e-03, -9.6645e-02, 5.7554e-02, 3.0361e-02,\n",
+ " 5.6569e-02, 1.1550e-01, -8.5696e-02, -1.2037e-01, 1.4408e-01,\n",
+ " -6.9288e-02, -9.6466e-02, -1.0642e-01, 6.7118e-02, -4.6111e-02,\n",
+ " 1.2625e-01, -2.2200e-02, 1.8784e-01, 5.6559e-02, -9.2792e-03,\n",
+ " 1.0988e-01, -8.4376e-02, 2.0674e-02, 5.1443e-02, -6.3788e-02,\n",
+ " 9.2447e-02, 5.4876e-03, -5.5736e-02, 1.1064e-01, -9.2663e-02,\n",
+ " -1.3774e-02, -1.1738e-02, 1.1746e-01, -1.1146e-01, 5.0254e-02,\n",
+ " -2.7132e-02, 6.3535e-02, -5.7441e-02, 1.4019e-01, 1.5358e-01,\n",
+ " -9.0577e-02, -6.1400e-02, -8.2789e-03, -3.0574e-02, -1.6684e-01,\n",
+ " 2.1997e-02, 1.2047e-01, -8.1132e-02, 2.7643e-02, -4.1913e-02,\n",
+ " 7.8445e-02, 3.4260e-02, 1.9198e-02, 8.3587e-03, 8.4882e-02,\n",
+ " 4.7922e-02, 3.2085e-03, 6.3019e-02, -1.2356e-01, -2.2035e-02,\n",
+ " -1.4141e-01, 3.0651e-02, -4.9958e-02, 2.0171e-02, -1.0585e-01,\n",
+ " -1.4497e-01, -4.1225e-02, -6.8969e-02, -1.5650e-01, -8.4516e-02,\n",
+ " -7.2454e-02, -1.9754e-02, -2.0815e-02, -1.4435e-01, -1.3507e-01,\n",
+ " -9.3353e-03, -7.8647e-02, 4.9376e-02, -9.1362e-02, -2.6405e-04,\n",
+ " -1.1060e-01, 6.1339e-02, -1.6488e-01, -1.2586e-02, -1.6126e-01,\n",
+ " -5.5025e-02, -3.0095e-02, -6.0076e-02, -8.4154e-02, -4.5846e-02,\n",
+ " 1.0521e-01, -1.0573e-01, -5.6448e-03, -1.1787e-01, -1.6642e-01,\n",
+ " -4.6485e-02, 2.1018e-03, -1.7721e-01, -2.3989e-02, 2.2526e-03,\n",
+ " -3.3936e-02, -8.5082e-04, 2.3318e-02, -1.2200e-01, -3.8985e-02,\n",
+ " -5.0310e-02, -2.1276e-01, -1.0934e-01, 5.0431e-02, 1.5243e-01,\n",
+ " -4.5213e-02, 5.6250e-02, -5.1513e-02, -7.0380e-02, 1.3558e-02,\n",
+ " 4.8311e-02, -1.0758e-01, -6.9672e-02, 4.1720e-02, -3.7519e-02,\n",
+ " 4.4889e-03, 3.1567e-02, 1.8889e-01, -2.1389e-02, -1.7544e-01,\n",
+ " 4.2167e-02, 2.3926e-01, -8.3352e-02, 2.3343e-02, 1.1568e-01,\n",
+ " -4.0584e-02, 4.9951e-02, 3.1471e-02, 5.3396e-02, 4.5164e-02,\n",
+ " 9.5883e-02, -1.0847e-01, -1.0457e-01, 4.1262e-02, 6.4844e-02,\n",
+ " 9.8069e-02, 1.5135e-02, -2.5175e-02, 7.6218e-02, -6.0823e-02,\n",
+ " -1.1629e-01, 6.7468e-03, 2.0872e-02, 1.2584e-01, -3.4727e-02,\n",
+ " -1.0161e-01, 8.7080e-02, -2.7376e-04, -4.8307e-02, -1.0124e-01,\n",
+ " -7.6946e-02, -1.9775e-02, -4.2434e-02, 3.4611e-02, 7.3197e-02,\n",
+ " 1.0860e-02, -1.0779e-02, -1.1027e-01, -2.7294e-02, 2.2437e-02,\n",
+ " -1.6622e-01, -2.6583e-02, 5.8286e-02, 2.9824e-03, -5.5497e-02,\n",
+ " 5.0274e-02, 5.9907e-02, 1.5858e-01, -2.4947e-02, 7.6584e-02,\n",
+ " 8.5319e-02, 3.4722e-03, 1.0288e-01, 1.0969e-01, 6.5799e-02,\n",
+ " -2.6551e-02, -1.7372e-01, 1.4605e-01, 2.1898e-02, -3.5002e-03,\n",
+ " -1.5310e-01, -3.0351e-02, 5.7421e-02, -1.3553e-01, -9.7861e-02,\n",
+ " 1.3477e-01, -1.2629e-01, -8.7702e-02, -4.4454e-03, -7.2945e-03,\n",
+ " -7.9274e-02, -1.0867e-02, 3.8146e-02, 1.0687e-01, -2.5816e-03,\n",
+ " 1.6793e-02, 6.4059e-02, -7.5229e-02, -8.1442e-02, 8.3586e-03,\n",
+ " -8.0715e-02, 8.9084e-03, -1.0933e-01, -1.4269e-02, -7.3607e-03,\n",
+ " 1.5704e-02, 9.4702e-03, 2.2811e-02, -8.7653e-02, 9.0603e-02,\n",
+ " 6.9251e-02, -3.1635e-02, 9.3687e-03, 5.9566e-03, -1.1568e-02,\n",
+ " 7.7497e-02, 8.1724e-03, 7.5156e-02, 5.9760e-02, -9.0665e-02,\n",
+ " -4.1795e-02, -5.5084e-02, -2.1276e-02, -7.5714e-03, 8.1334e-02,\n",
+ " 3.2634e-02, -6.9134e-02, 7.7450e-02, -6.0068e-02, 1.4064e-01,\n",
+ " -1.5251e-01, -5.5206e-02, 9.4398e-02, 6.7102e-02, -2.1777e-02,\n",
+ " -1.0860e-01, 8.1245e-02, -7.5645e-02, 1.3485e-02, 8.9177e-02,\n",
+ " 1.7675e-02, -8.5894e-02, -1.2788e-02, 2.8444e-02, 1.0189e-01,\n",
+ " 1.3065e-01, 4.1666e-02, 4.3118e-02, -4.6221e-03, -8.9139e-02,\n",
+ " 7.8242e-02, 6.2911e-02, -2.1091e-01, -8.0931e-02, 3.9291e-02,\n",
+ " 2.0712e-01, -1.2777e-02, 3.2420e-02, 3.9909e-02, -4.8477e-02,\n",
+ " 1.1958e-01, -6.7129e-02, -5.5796e-02, -7.8439e-02, 1.0222e-01,\n",
+ " -1.2084e-01, 5.1758e-02, 1.8989e-01, -2.1552e-01, 1.0309e-01,\n",
+ " 1.4989e-01, 2.9901e-02, 5.6023e-02, -7.9886e-03, 4.2809e-02,\n",
+ " 7.0390e-02, 6.5856e-02, 1.8404e-02, -1.8389e-01, 3.2689e-02,\n",
+ " -2.9534e-02, -7.4516e-02, 1.9053e-02, 6.8197e-03, -8.6406e-02,\n",
+ " -1.2057e-01, 5.4083e-02, -2.8486e-02, 2.0098e-03, -2.4764e-02,\n",
+ " -6.8012e-02, -9.7861e-02, 2.7055e-02, 3.1677e-02, 1.7974e-02,\n",
+ " 5.6053e-02, -1.2744e-02, 1.1241e-02, 1.1206e-01, -3.9609e-02,\n",
+ " 1.8022e-02, -3.5299e-02, 4.1814e-02, -7.1772e-02, -7.2119e-02,\n",
+ " 3.1803e-02, -5.6058e-02, -1.2408e-01, 1.1487e-01, 8.2424e-02,\n",
+ " -7.6738e-02, 3.6102e-02, 1.0330e-01, 1.6182e-01, 7.3596e-02,\n",
+ " -8.2679e-02, 2.3563e-02, -4.7735e-02, 5.1281e-02, -4.4526e-02,\n",
+ " -1.8515e-02, 1.0371e-02, -2.1650e-02, 1.7475e-02, -6.6999e-02,\n",
+ " -3.5237e-03, -6.6710e-03, 4.5694e-02, -2.4330e-02, -9.4324e-02,\n",
+ " 1.0351e-01, -9.7201e-02, 9.9928e-02, 6.0018e-03, -1.9398e-01,\n",
+ " -6.9612e-02, 3.7000e-02, 2.5857e-02, -2.0493e-01, 1.0143e-01,\n",
+ " 2.9652e-02, -1.0923e-01, -1.3063e-01, 8.0357e-02, -2.8222e-02,\n",
+ " 1.1236e-01, 1.0068e-01, 1.8715e-01, -4.7519e-03, -1.9877e-02,\n",
+ " -5.8805e-02, -2.2623e-02, -9.1535e-02, -3.9104e-02, 1.1634e-02,\n",
+ " -1.2658e-02, -1.9400e-01, 1.3103e-01, 3.6966e-02, -6.6630e-02,\n",
+ " 3.6699e-02, -1.3168e-01, -3.5448e-02, -6.5795e-02, 3.8508e-02,\n",
+ " -2.8908e-02, 5.9616e-03, -1.1947e-01, -4.5128e-02, 2.2609e-02,\n",
+ " -6.0078e-02, 1.2044e-02, -4.2161e-02, -1.7942e-01, -1.4760e-01,\n",
+ " -2.2213e-02, -1.2489e-01, 1.2302e-01, 2.6101e-02, 1.1545e-01,\n",
+ " -6.0973e-02, -9.3216e-02, -1.3401e-01, 6.9192e-02, 1.0547e-01,\n",
+ " 4.1468e-02, 7.5163e-02, -4.5361e-02, 2.8214e-02, -9.8323e-02,\n",
+ " -8.9710e-04, 7.1415e-02, -8.6964e-02, 7.2207e-02, -5.4545e-02,\n",
+ " -4.4864e-02, -1.0401e-02, 1.1054e-02, -5.4850e-03, -1.5119e-01,\n",
+ " 2.7383e-02, 7.9438e-02, 9.7938e-02, -7.6253e-02, -9.8225e-02,\n",
+ " -3.6008e-03, 2.5735e-02, -4.4595e-02, 9.2904e-03, 2.4263e-02,\n",
+ " -1.1514e-02, -2.5885e-02, 9.3242e-02, 3.7527e-02, 1.4858e-03,\n",
+ " 4.4235e-02, 4.6888e-02, 2.9045e-04, -3.7129e-02, -1.3034e-01,\n",
+ " -1.3929e-01, 6.9240e-02, -2.5642e-02, -8.9732e-03, -1.2048e-01,\n",
+ " 5.1459e-02, -5.1739e-02, -8.8432e-02, -6.8968e-02, -1.3449e-01,\n",
+ " 1.8770e-02, 1.4933e-01, -1.4537e-01, -1.3368e-02, -2.0281e-03])\n"
+ ]
+ }
+ ],
+ "source": [
+ "o = 10800\n",
+ "print(conc[o:o+1000])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "eqOXilqMs-yF",
+ "outputId": "06799a3b-983b-4f51-a7bd-a901c041bd05"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.317379 [ 0/735856]\n",
+ "loss: 1.467329 [12800/735856]\n",
+ "loss: 0.863656 [25600/735856]\n",
+ "loss: 0.814204 [38400/735856]\n",
+ "loss: 0.746839 [51200/735856]\n",
+ "loss: 0.734121 [64000/735856]\n",
+ "loss: 0.761491 [76800/735856]\n",
+ "loss: 0.680998 [89600/735856]\n",
+ "loss: 0.731149 [102400/735856]\n",
+ "loss: 0.826501 [115200/735856]\n",
+ "loss: 0.662327 [128000/735856]\n",
+ "loss: 0.612664 [140800/735856]\n",
+ "loss: 0.755015 [153600/735856]\n",
+ "loss: 0.400373 [166400/735856]\n",
+ "loss: 0.558040 [179200/735856]\n",
+ "loss: 0.603362 [192000/735856]\n",
+ "loss: 0.418064 [204800/735856]\n",
+ "loss: 0.628256 [217600/735856]\n",
+ "loss: 0.377127 [230400/735856]\n",
+ "loss: 0.420045 [243200/735856]\n",
+ "loss: 0.558597 [256000/735856]\n",
+ "loss: 0.438556 [268800/735856]\n",
+ "loss: 0.684690 [281600/735856]\n",
+ "loss: 0.590059 [294400/735856]\n",
+ "loss: 0.557874 [307200/735856]\n",
+ "loss: 0.494909 [320000/735856]\n",
+ "loss: 0.617219 [332800/735856]\n",
+ "loss: 0.351243 [345600/735856]\n",
+ "loss: 0.454522 [358400/735856]\n",
+ "loss: 0.429664 [371200/735856]\n",
+ "loss: 0.468215 [384000/735856]\n",
+ "loss: 0.401258 [396800/735856]\n",
+ "loss: 0.474102 [409600/735856]\n",
+ "loss: 0.562686 [422400/735856]\n",
+ "loss: 0.483383 [435200/735856]\n",
+ "loss: 0.348151 [448000/735856]\n",
+ "loss: 0.528455 [460800/735856]\n",
+ "loss: 0.545382 [473600/735856]\n",
+ "loss: 0.370390 [486400/735856]\n",
+ "loss: 0.567488 [499200/735856]\n",
+ "loss: 0.480258 [512000/735856]\n",
+ "loss: 0.605147 [524800/735856]\n",
+ "loss: 0.415480 [537600/735856]\n",
+ "loss: 0.427186 [550400/735856]\n",
+ "loss: 0.391611 [563200/735856]\n",
+ "loss: 0.604008 [576000/735856]\n",
+ "loss: 0.745116 [588800/735856]\n",
+ "loss: 0.357690 [601600/735856]\n",
+ "loss: 0.347922 [614400/735856]\n",
+ "loss: 0.518981 [627200/735856]\n",
+ "loss: 0.435502 [640000/735856]\n",
+ "loss: 0.387361 [652800/735856]\n",
+ "loss: 0.449226 [665600/735856]\n",
+ "loss: 0.447640 [678400/735856]\n",
+ "loss: 0.403468 [691200/735856]\n",
+ "loss: 0.349100 [704000/735856]\n",
+ "loss: 0.361501 [716800/735856]\n",
+ "loss: 0.315695 [729600/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.0%, Avg loss: 0.434803 \n",
+ "\n",
+ "loss: 0.436410 [ 0/735856]\n",
+ "loss: 0.429430 [12800/735856]\n",
+ "loss: 0.300599 [25600/735856]\n",
+ "loss: 0.416907 [38400/735856]\n",
+ "loss: 0.505084 [51200/735856]\n",
+ "loss: 0.399249 [64000/735856]\n",
+ "loss: 0.379393 [76800/735856]\n",
+ "loss: 0.375960 [89600/735856]\n",
+ "loss: 0.408403 [102400/735856]\n",
+ "loss: 0.499817 [115200/735856]\n",
+ "loss: 0.450748 [128000/735856]\n",
+ "loss: 0.408854 [140800/735856]\n",
+ "loss: 0.342387 [153600/735856]\n",
+ "loss: 0.330833 [166400/735856]\n",
+ "loss: 0.416435 [179200/735856]\n",
+ "loss: 0.340663 [192000/735856]\n",
+ "loss: 0.408621 [204800/735856]\n",
+ "loss: 0.444404 [217600/735856]\n",
+ "loss: 0.453196 [230400/735856]\n",
+ "loss: 0.408210 [243200/735856]\n",
+ "loss: 0.460274 [256000/735856]\n",
+ "loss: 0.334112 [268800/735856]\n",
+ "loss: 0.330720 [281600/735856]\n",
+ "loss: 0.316345 [294400/735856]\n",
+ "loss: 0.248728 [307200/735856]\n",
+ "loss: 0.464760 [320000/735856]\n",
+ "loss: 0.427282 [332800/735856]\n",
+ "loss: 0.431015 [345600/735856]\n",
+ "loss: 0.491930 [358400/735856]\n",
+ "loss: 0.379011 [371200/735856]\n",
+ "loss: 0.336299 [384000/735856]\n",
+ "loss: 0.312829 [396800/735856]\n",
+ "loss: 0.355771 [409600/735856]\n",
+ "loss: 0.289162 [422400/735856]\n",
+ "loss: 0.583171 [435200/735856]\n",
+ "loss: 0.499083 [448000/735856]\n",
+ "loss: 0.423254 [460800/735856]\n",
+ "loss: 0.436303 [473600/735856]\n",
+ "loss: 0.360267 [486400/735856]\n",
+ "loss: 0.376950 [499200/735856]\n",
+ "loss: 0.424678 [512000/735856]\n",
+ "loss: 0.381343 [524800/735856]\n",
+ "loss: 0.429872 [537600/735856]\n",
+ "loss: 0.355957 [550400/735856]\n",
+ "loss: 0.409392 [563200/735856]\n",
+ "loss: 0.352608 [576000/735856]\n",
+ "loss: 0.265125 [588800/735856]\n",
+ "loss: 0.406446 [601600/735856]\n",
+ "loss: 0.368569 [614400/735856]\n",
+ "loss: 0.437190 [627200/735856]\n",
+ "loss: 0.305703 [640000/735856]\n",
+ "loss: 0.347399 [652800/735856]\n",
+ "loss: 0.331695 [665600/735856]\n",
+ "loss: 0.457639 [678400/735856]\n",
+ "loss: 0.473799 [691200/735856]\n",
+ "loss: 0.489939 [704000/735856]\n",
+ "loss: 0.370199 [716800/735856]\n",
+ "loss: 0.331481 [729600/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.6%, Avg loss: 0.387414 \n",
+ "\n",
+ "loss: 0.443714 [ 0/735856]\n",
+ "loss: 0.449352 [12800/735856]\n",
+ "loss: 0.414199 [25600/735856]\n",
+ "loss: 0.404422 [38400/735856]\n",
+ "loss: 0.368516 [51200/735856]\n",
+ "loss: 0.346299 [64000/735856]\n",
+ "loss: 0.484283 [76800/735856]\n",
+ "loss: 0.401498 [89600/735856]\n",
+ "loss: 0.410465 [102400/735856]\n",
+ "loss: 0.273847 [115200/735856]\n",
+ "loss: 0.399912 [128000/735856]\n",
+ "loss: 0.479173 [140800/735856]\n",
+ "loss: 0.401442 [153600/735856]\n",
+ "loss: 0.285392 [166400/735856]\n",
+ "loss: 0.421379 [179200/735856]\n",
+ "loss: 0.279620 [192000/735856]\n",
+ "loss: 0.390798 [204800/735856]\n",
+ "loss: 0.212120 [217600/735856]\n",
+ "loss: 0.354751 [230400/735856]\n",
+ "loss: 0.219936 [243200/735856]\n",
+ "loss: 0.402732 [256000/735856]\n",
+ "loss: 0.519650 [268800/735856]\n",
+ "loss: 0.288248 [281600/735856]\n",
+ "loss: 0.384210 [294400/735856]\n",
+ "loss: 0.478157 [307200/735856]\n",
+ "loss: 0.439406 [320000/735856]\n",
+ "loss: 0.389753 [332800/735856]\n",
+ "loss: 0.356618 [345600/735856]\n",
+ "loss: 0.323311 [358400/735856]\n",
+ "loss: 0.398488 [371200/735856]\n",
+ "loss: 0.319226 [384000/735856]\n",
+ "loss: 0.332842 [396800/735856]\n",
+ "loss: 0.252516 [409600/735856]\n",
+ "loss: 0.293284 [422400/735856]\n",
+ "loss: 0.334755 [435200/735856]\n",
+ "loss: 0.377591 [448000/735856]\n",
+ "loss: 0.380793 [460800/735856]\n",
+ "loss: 0.403311 [473600/735856]\n",
+ "loss: 0.357482 [486400/735856]\n",
+ "loss: 0.304435 [499200/735856]\n",
+ "loss: 0.241090 [512000/735856]\n",
+ "loss: 0.324607 [524800/735856]\n",
+ "loss: 0.328274 [537600/735856]\n",
+ "loss: 0.244099 [550400/735856]\n",
+ "loss: 0.415031 [563200/735856]\n",
+ "loss: 0.348006 [576000/735856]\n",
+ "loss: 0.283688 [588800/735856]\n",
+ "loss: 0.300562 [601600/735856]\n",
+ "loss: 0.326958 [614400/735856]\n",
+ "loss: 0.369248 [627200/735856]\n",
+ "loss: 0.332926 [640000/735856]\n",
+ "loss: 0.302038 [652800/735856]\n",
+ "loss: 0.342727 [665600/735856]\n",
+ "loss: 0.368321 [678400/735856]\n",
+ "loss: 0.231897 [691200/735856]\n",
+ "loss: 0.431953 [704000/735856]\n",
+ "loss: 0.248471 [716800/735856]\n",
+ "loss: 0.294091 [729600/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.0%, Avg loss: 0.366894 \n",
+ "\n",
+ "loss: 0.376553 [ 0/735856]\n",
+ "loss: 0.220013 [12800/735856]\n",
+ "loss: 0.282281 [25600/735856]\n",
+ "loss: 0.397939 [38400/735856]\n",
+ "loss: 0.286681 [51200/735856]\n",
+ "loss: 0.299091 [64000/735856]\n",
+ "loss: 0.373950 [76800/735856]\n",
+ "loss: 0.429636 [89600/735856]\n",
+ "loss: 0.306355 [102400/735856]\n",
+ "loss: 0.411843 [115200/735856]\n",
+ "loss: 0.228846 [128000/735856]\n",
+ "loss: 0.207205 [140800/735856]\n",
+ "loss: 0.351640 [153600/735856]\n",
+ "loss: 0.401937 [166400/735856]\n",
+ "loss: 0.273460 [179200/735856]\n",
+ "loss: 0.327492 [192000/735856]\n",
+ "loss: 0.331937 [204800/735856]\n",
+ "loss: 0.228119 [217600/735856]\n",
+ "loss: 0.287975 [230400/735856]\n",
+ "loss: 0.224747 [243200/735856]\n",
+ "loss: 0.283936 [256000/735856]\n",
+ "loss: 0.342105 [268800/735856]\n",
+ "loss: 0.292306 [281600/735856]\n",
+ "loss: 0.292342 [294400/735856]\n",
+ "loss: 0.240786 [307200/735856]\n",
+ "loss: 0.313040 [320000/735856]\n",
+ "loss: 0.419340 [332800/735856]\n",
+ "loss: 0.489452 [345600/735856]\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(10):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ "\n",
+ " if batch % 100 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "4P-VA0vcs-yH"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"/results:128:\"+str(lr)+\".pkl\", \"wb\") as f:\n",
+ " pickle.dump(stats, f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "641-b_VCvT2b",
+ "outputId": "cced38ab-5c04-45b2-faf4-e73327126159"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "F_OKqiiHs-yJ",
+ "outputId": "65786b88-05f4-42fa-a851-03397ef4457a"
+ },
+ "outputs": [],
+ "source": [
+ "lrs = [0.1, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00001]\n",
+ "for l in lrs:\n",
+ " with open(train_dir+\"/results:128:\"+str(l)+\".pkl\", \"rb\") as f:\n",
+ " res = pickle.load(f)\n",
+ " print(str(l)+\": \" + str(np.amax(res[\"test\"], axis=0)))#+ str(np.max(res[\"test\"]))\n",
+ " # print(str(l)+\": \" + str(res[\"test\"]))\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "rADw-XkfKjOo",
+ "outputId": "06c54a2c-f7c2-4610-f879-3e1c2f98543f"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.1: [[0, 3.6898819485246297, 4.914933837429111], [1, 3.695287103771977, 4.914933837429111], [2, 3.691172517592003, 5.505213732544667], [3, 3.6920483804158226, 4.967376059515824], [4, 3.6939517986755845, 5.505213732544667], [5, 3.6917366434742993, 4.914933837429111], [6, 3.695435837910811, 5.505213732544667], [7, 3.6978058357506574, 5.135679004817367], [8, 3.6948341036363623, 5.505213732544667], [9, 3.6921330658768343, 5.135679004817367]]\n",
+ "0.01: [[0, 0.5728003431500958, 81.8086468687115], [1, 0.5517885946725348, 82.61967193121532], [2, 3.984756194857093, 25.844258796268065], [3, 0.5739870879932797, 81.7476675407037], [4, 0.7832032613188912, 75.77779132873955], [5, 0.7142617320772638, 77.80474419171901], [6, 0.6602287095348103, 79.28654186230868], [7, 0.6738644539380036, 79.2719068235868], [8, 0.6469118589079138, 79.77071772669065], [9, 0.6788249858734946, 79.28898103542899]]\n",
+ "0.005: [[0, 0.4834194714537649, 83.79047502896519], [1, 0.466142692822562, 84.33928898103544], [2, 0.4559767278791776, 84.9515214342338], [3, 0.4488265364432298, 84.86493078846271], [4, 0.4554814101660307, 84.773461796451], [5, 0.5149212768315897, 83.05872309287152], [6, 0.4551808235472338, 84.86127202878224], [7, 0.4531376465992325, 85.06494298432831], [8, 0.4589428385362238, 84.83078236477834], [9, 0.4409179601951992, 85.40642722117202]]\n",
+ "0.001: [[0, 0.4104761779773254, 85.89670101835478], [1, 0.36889259526491536, 87.17604731995854], [2, 0.3517718464717292, 87.6992499542655], [3, 0.35526543692939927, 87.57607171168974], [4, 0.3493265717198808, 87.76266845539362], [5, 0.35079776836259874, 87.47362644063662], [6, 0.34534544340812845, 87.96268065125923], [7, 0.35734797465540874, 87.72608085858894], [8, 0.3524193228360457, 87.63339228001708], [9, 0.35447056082407136, 88.0382950179889]]\n",
+ "0.0005: [[0, 0.4435185831906085, 85.1039697542533], [1, 0.37539843543085405, 86.94310628696871], [2, 0.35873422210283473, 87.3797182755046], [3, 0.34818319706667605, 87.93097140069517], [4, 0.34545205666010914, 87.86633331300689], [5, 0.3371337376732536, 88.10415269223734], [6, 0.33852135716659976, 88.11512897127874], [7, 0.33852605533302293, 88.14074028904201], [8, 0.33997187332225476, 88.21025672297091], [9, 0.3402654077747311, 88.1968412708092]]\n",
+ "0.0001: [[0, 0.6723326555745278, 79.72437343740472], [1, 0.5084800024207409, 83.89901823281907], [2, 0.45863669222676995, 84.88932251966584], [3, 0.42524330169194946, 85.90767729739618], [4, 0.4028480564841242, 86.33575218001097], [5, 0.38621816764383715, 86.9126166229648], [6, 0.3782781209337544, 87.11506799195074], [7, 0.3759017101781045, 87.00530520153667], [8, 0.3668581307538772, 87.32117812061712], [9, 0.3569657983208967, 87.71754375266785]]\n",
+ "1e-05: [[0, 1.8116753936371826, 54.021586682114766], [1, 1.3575628893609724, 63.8148667601683], [2, 1.0996285610935432, 69.48716385145435], [3, 0.9349309672803477, 73.7788889566437], [4, 0.8294046315685635, 76.08878590157937], [5, 0.7614829346374863, 77.91938532837368], [6, 0.7032811826737176, 79.30727483383133], [7, 0.6657257149818349, 80.09024940545156], [8, 0.6363296165202226, 80.82931886090616], [9, 0.6094131586890139, 81.52570278675529]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "lrs = [0.1, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00001]\n",
+ "for l in lrs:\n",
+ " with open(train_dir+\"/results:128:\"+str(l)+\".pkl\", \"rb\") as f:\n",
+ " res = pickle.load(f)\n",
+ " # print(str(l)+\": \" + str(np.amax(res[\"test\"], axis=0)))#+ str(np.max(res[\"test\"]))\n",
+ " print(str(l)+\": \" + str(res[\"test\"]))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "HGpNYzG_s-yJ",
+ "outputId": "783622a5-249f-4dd8-d242-fc6dfa47443c"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using cache found in /home/jeffrey/.cache/torch/hub/pytorch_vision_v0.10.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "import torch\n",
+ "resnet = torch.hub.load('pytorch/vision:v0.10.0', 'resnet18', pretrained=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "uZFgT6wss-yL",
+ "outputId": "10f8fc51-abb7-4c2b-f608-85229f3de29d",
+ "tags": []
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "11699132\n"
+ ]
+ }
+ ],
+ "source": [
+ "total = 0\n",
+ "for i in resnet.state_dict().values():\n",
+ " total += i.flatten().size(dim=0)\n",
+ "print(total)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Optimizer analysis <a class=\"anchor\" id=\"optim\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "id": "mRZYP5UNs-yL"
+ },
+ "outputs": [],
+ "source": [
+ "# internal state test\n",
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
+ "old = model.state_dict()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0877, -0.1623, -0.0757, -0.1486, 0.1212, 0.1070, 0.0221, -0.1306,\n",
+ " 0.0798, -0.1525, -0.0297, -0.1715, 0.1039, 0.0143, 0.0982, 0.0428,\n",
+ " -0.0983, -0.0698, 0.1894, 0.1400, 0.0139, -0.0640, 0.0410, -0.0332,\n",
+ " -0.0993, -0.0840, -0.1224, 0.0723, 0.1994, 0.0017, -0.1309, 0.0044])"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "old[\"conv1.bias\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1695, 0.0365, 0.0043, -0.0058, 0.1130, 0.1614, -0.1921, 0.0229,\n",
+ " 0.1472, 0.0111, -0.1327, -0.0368, 0.0536, -0.0637, 0.1539, 0.1022,\n",
+ " 0.1948, -0.1443, 0.1046, 0.1746, 0.1998, -0.0572, 0.0675, -0.1533,\n",
+ " -0.1863, -0.0397, 0.1823, -0.0121, 0.0045, 0.0704, 0.1362, 0.1068])"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "model.state_dict()[\"conv1.bias\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0877, -0.1623, -0.0757, -0.1486, 0.1212, 0.1070, 0.0221, -0.1306,\n",
+ " 0.0798, -0.1525, -0.0297, -0.1715, 0.1039, 0.0143, 0.0982, 0.0428,\n",
+ " -0.0983, -0.0698, 0.1894, 0.1400, 0.0139, -0.0640, 0.0410, -0.0332,\n",
+ " -0.0993, -0.0840, -0.1224, 0.0723, 0.1994, 0.0017, -0.1309, 0.0044])"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "old[\"conv1.bias\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'state': {},\n",
+ " 'param_groups': [{'lr': 0.0005,\n",
+ " 'betas': (0.9, 0.999),\n",
+ " 'eps': 1e-08,\n",
+ " 'weight_decay': 0,\n",
+ " 'amsgrad': False,\n",
+ " 'params': [0, 1, 2, 3, 4, 5, 6, 7]}]}"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimizer.state_dict()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n"
+ ]
+ }
+ ],
+ "source": [
+ "for p in model.parameters():\n",
+ " print(type(p))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "8"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(optimizer.param_groups[0][\"params\"])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "IndexError",
+ "evalue": "list index out of range",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)",
+ "Input \u001b[0;32mIn [19]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43moptimizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparam_groups\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\n",
+ "\u001b[0;31mIndexError\u001b[0m: list index out of range"
+ ]
+ }
+ ],
+ "source": [
+ "optimizer.param_groups[1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "dict_keys(['params', 'lr', 'betas', 'eps', 'weight_decay', 'amsgrad'])"
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimizer.param_groups[0].keys()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Parameter containing:\n",
+ "tensor([-0.0877, -0.1623, -0.0757, -0.1486, 0.1212, 0.1070, 0.0221, -0.1306,\n",
+ " 0.0798, -0.1525, -0.0297, -0.1715, 0.1039, 0.0143, 0.0982, 0.0428,\n",
+ " -0.0983, -0.0698, 0.1894, 0.1400, 0.0139, -0.0640, 0.0410, -0.0332,\n",
+ " -0.0993, -0.0840, -0.1224, 0.0723, 0.1994, 0.0017, -0.1309, 0.0044],\n",
+ " requires_grad=True)"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimizer.param_groups[0][\"params\"][1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# --> yes the optimizer values do not get updates"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "defaultdict(dict, {})"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimizer.state"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# optimizer.state is a dictionary that gets filled during the first step() call\n",
+ "# as keys it has the params and as values it has the internal state of the optimizer (first momentum, second momentum etc)\n",
+ "# stored the values in vals, they are from running the training loop"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "8"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(vals)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vals_list = list(vals)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "8"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(vals_list)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'step': 69,\n",
+ " 'exp_avg': tensor([-1.4137e-03, -1.0432e-02, -5.7605e-03, -1.5292e-02, -1.1802e-02,\n",
+ " 1.1299e-03, 2.1533e-03, -9.7591e-03, -8.8733e-03, -4.7788e-03,\n",
+ " -1.9228e-03, -5.7594e-03, -5.4949e-05, -3.8590e-05, 1.3072e-04,\n",
+ " -7.8018e-03, -6.1446e-04, -2.9151e-03, -3.3301e-03, -2.0083e-03,\n",
+ " -3.0533e-03, -3.5316e-04, -8.1218e-03, 5.7864e-04, 5.8342e-04,\n",
+ " -1.1397e-02, -8.2111e-04, -6.8639e-03, -7.7449e-04, 1.0854e-04,\n",
+ " -4.7743e-05, -9.0613e-03]),\n",
+ " 'exp_avg_sq': tensor([9.6760e-06, 2.4495e-05, 8.1120e-06, 3.0419e-05, 1.0932e-05, 2.3003e-05,\n",
+ " 1.9296e-05, 1.5492e-05, 2.6551e-06, 1.1472e-05, 2.6787e-05, 9.0655e-05,\n",
+ " 8.7915e-11, 3.1380e-05, 4.9582e-06, 3.9729e-06, 7.5247e-06, 1.8417e-05,\n",
+ " 6.9078e-06, 2.9552e-05, 5.0895e-06, 1.6462e-06, 2.1158e-06, 2.8078e-06,\n",
+ " 2.5839e-06, 2.6732e-05, 2.0372e-05, 9.5084e-07, 2.0701e-06, 1.2862e-06,\n",
+ " 1.5106e-06, 2.5722e-05])}"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "vals_list[1] # entry for "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The most feasible solution would be to create a new optimizer and then \n",
+ "# vals = list(optimizer.state.values())\n",
+ "# create new optimizer\n",
+ "# for i, k in enmumerate(optimizer.param_groups[0][\"params\"]):\n",
+ "# optimizer.state[k] = vals[i]\n",
+ "# https://pytorch.org/docs/stable/_modules/torch/optim/adam.html#Adam\n",
+ "# https://pytorch.org/docs/stable/_modules/torch/optim/optimizer.html#Optimizer"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## FFT <a class=\"anchor\" id=\"fft\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(735856, 1, 28, 28)"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train[\"train_x\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "5748.875"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "735856 / 128"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "weights = {}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.124378 [ 0/735856]\n",
+ "loss: 1.539611 [64000/735856]\n",
+ "loss: 0.719579 [128000/735856]\n",
+ "loss: 0.685157 [192000/735856]\n",
+ "loss: 0.778637 [256000/735856]\n",
+ "loss: 0.493262 [320000/735856]\n",
+ "loss: 0.423785 [384000/735856]\n",
+ "loss: 0.531239 [448000/735856]\n",
+ "loss: 0.803173 [512000/735856]\n",
+ "loss: 0.498672 [576000/735856]\n",
+ "loss: 0.453685 [640000/735856]\n",
+ "loss: 0.355350 [704000/735856]\n",
+ "loss: 0.417364 [768000/735856]\n",
+ "loss: 0.462418 [832000/735856]\n",
+ "loss: 0.361217 [896000/735856]\n",
+ "loss: 0.484760 [960000/735856]\n",
+ "loss: 0.360997 [1024000/735856]\n",
+ "loss: 0.353997 [1088000/735856]\n",
+ "loss: 0.378490 [1152000/735856]\n",
+ "loss: 0.376164 [1216000/735856]\n",
+ "loss: 0.375268 [1280000/735856]\n",
+ "loss: 0.570408 [1344000/735856]\n",
+ "loss: 0.295247 [1408000/735856]\n",
+ "loss: 0.257762 [1472000/735856]\n",
+ "loss: 0.609368 [1536000/735856]\n",
+ "loss: 0.423437 [1600000/735856]\n",
+ "loss: 0.363265 [1664000/735856]\n",
+ "loss: 0.393251 [1728000/735856]\n",
+ "loss: 0.353971 [1792000/735856]\n",
+ "loss: 0.279443 [1856000/735856]\n",
+ "loss: 0.532804 [1920000/735856]\n",
+ "loss: 0.364327 [1984000/735856]\n",
+ "loss: 0.310962 [2048000/735856]\n",
+ "loss: 0.306962 [2112000/735856]\n",
+ "loss: 0.391289 [2176000/735856]\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "batch = 0\n",
+ "for e in range(3):\n",
+ " #training\n",
+ " for X, y in train_dataloader:\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " weight = {}\n",
+ " for k,v in model.state_dict().items():\n",
+ " weight[k] = v.clone()\n",
+ " \n",
+ " weights[str(batch)] = weight\n",
+ " batch += 1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.2%, Avg loss: 0.367197 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "size = len(test_dataloader.dataset)\n",
+ "num_batches = len(test_dataloader)\n",
+ "model.eval()\n",
+ "test_loss, correct = 0, 0\n",
+ "with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ "test_loss /= num_batches\n",
+ "correct /= size\n",
+ "print(\"epoch:\")\n",
+ "print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ "stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "dict_keys(['0', '500', '1000', '1500', '2000', '2500', '3000', '3500', '4000', '4500', '5000', '5500', '6000', '6500', '7000', '7500', '8000', '8500', '9000', '9500', '10000', '10500', '11000', '11500', '12000', '12500', '13000', '13500', '14000', '14500', '15000', '15500', '16000', '16500', '17000'])"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "weights.keys()"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "with open(\"vecs.pkl\", \"wb\") as f:\n",
+ " \n",
+ " json.dump(weights, f)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "* 1d on flattend\n",
+ "* 1d on layers\n",
+ "* nd on layers"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# working on the random initialization\n",
+ "flat = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat.append(v.flatten())\n",
+ "conc = torch.cat(flat)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1690046"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(conc)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1440, -0.0482, 0.2070, -0.2534, -0.2413, 0.0336, -0.2401, 0.2761,\n",
+ " 0.2361, -0.2687])"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "conc[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch.fft as fft"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1690046"
+ ]
+ },
+ "execution_count": 29,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft = fft.fft(conc)\n",
+ "len(flat_fft)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "845024"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft = fft.rfft(conc)\n",
+ "len(flat_fft)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-3912.2754+0.0000j, -685.3215+117.2780j, -718.2836-68.4478j,\n",
+ " ..., 33.0949-6.6868j, 49.2176+6.2663j,\n",
+ " -9.9980+0.0000j])"
+ ]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse = fft.irfft(flat_fft)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1440, -0.0482, 0.2070, -0.2534, -0.2413, 0.0336, -0.2401, 0.2761,\n",
+ " 0.2361, -0.2687])"
+ ]
+ },
+ "execution_count": 33,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "reverse[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(0.0004)"
+ ]
+ },
+ "execution_count": 34,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc - reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "top10 = torch.zeros(flat_fft.size(dim=0), dtype = torch.cfloat)\n",
+ "top10[0:84502] = flat_fft[0:84502]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-3912.2754+0.0000j, -685.3215+117.2780j, -718.2836-68.4478j,\n",
+ " ..., 0.0000+0.0000j, 0.0000+0.0000j,\n",
+ " 0.0000+0.0000j])"
+ ]
+ },
+ "execution_count": 36,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "top10"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse_t10 = fft.irfft(top10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0439, -0.0400, -0.0357, -0.0312, -0.0269, -0.0229, -0.0193, -0.0162,\n",
+ " -0.0134, -0.0109])"
+ ]
+ },
+ "execution_count": 38,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "reverse_t10[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(40.2866)"
+ ]
+ },
+ "execution_count": 39,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_t10 - reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "d10 = torch.zeros(flat_fft.size(dim=0), dtype = torch.cfloat)\n",
+ "d10[-84502:] = flat_fft[-84502:]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse_d10 = fft.irfft(d10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0026, 0.0042, -0.0056, 0.0065, -0.0065, 0.0053, -0.0029, -0.0008,\n",
+ " 0.0059, -0.0120])"
+ ]
+ },
+ "execution_count": 42,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "reverse_d10[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(43.7672)"
+ ]
+ },
+ "execution_count": 43,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_d10 - reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 44,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rand = torch.rand(conc.size(dim=0))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 46,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(754.6744)"
+ ]
+ },
+ "execution_count": 46,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(rand - reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 47,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(29442.5273)"
+ ]
+ },
+ "execution_count": 47,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_d10 - reverse, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(27450.3555)"
+ ]
+ },
+ "execution_count": 48,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_t10 - reverse, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 49,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(30432.2695)"
+ ]
+ },
+ "execution_count": 49,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 50,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-3912.2754+0.0000j, -685.3215+117.2780j, -718.2836-68.4478j,\n",
+ " ..., 33.0949-6.6868j, 49.2176+6.2663j,\n",
+ " -9.9980+0.0000j])"
+ ]
+ },
+ "execution_count": 51,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([3912.2754, 695.2839, 721.5375, ..., 33.7637, 49.6149,\n",
+ " 9.9980])"
+ ]
+ },
+ "execution_count": 52,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft.abs()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import matplotlib.pyplot as plt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABgA0lEQVR4nO29eXgc5Z3g//l2S7ItW7Zl+UaWbIExRiaAZbA5AiYcARZiAiFckwkTzp1kd7Jz7DA5vF4nM0N2JjvJ/oZdMIQlk+VwwByGgYQj3NjGlmNiCeJLWLJ8W5Zl2bItdff7+6MOVVVXd1cf6m7J7+d59KiP6qq33nrr/b7fs0QphUaj0WhObkKFboBGo9FoCo8WBhqNRqPRwkCj0Wg0WhhoNBqNBi0MNBqNRoMWBhqNRqNBCwONZtAgIgtFpL3Q7dAMTbQw0OQFEdkuIsdE5IiI7BWRJ0RkVKHblQwRmS4iSkRKcrS/4SJySES+5PPdv4jIc7k4jkaTCVoYaPLJ9UqpUcBcYB7wg3R+LAaDZsx6hYhS6jiwHPhTz3Zh4Dbgl/lrnUbjZtDcWJqhg1JqJ/AaMEdEKkXkFRHZLyKd5utqa1sReUdE/l5EPgR6gDoR+TMR+UxEukWkRUTuc2y/UETaReS/isg+EdktIjeIyLUisllEDorI9xzbh0TkARHZJiIdIvJrERlnfv2e+f+QqdFcYP7mW+bxO0XktyJS69ifEpFvi8gWYIvP6f8SuElEyh2ffRnjXnwt2bl5MY91muP9EyLyY8f760Rkg6mNfCQiX3B897cistM8ziYRuTzRcTQnB1oYaPKOiEwDrgV+jzEG/y9QC9QAx4B/9fzkG8C9QAXQCuwDrgNGA38G/IuIzHVsPxkYDpwCLAYeBf4EaAC+CPxQRGaY2/4n4AbgUmAq0Ak8ZH53ifl/rFJqlFJqlYgsAr4H3AhMAN4Hnva09wZgPnCm99yVUh8Bu83fO8/vKaVUJMC5BUJEzgUeB+4DqoBHgJUiMkxEZgHfAc5TSlVgCKPt6R5DM8RQSuk//TfgfxiTzRHgEMaE/r+BET7bnQN0Ot6/AyxNse8Xgb8wXy/EEChh830FoID5ju0bgRvM158Blzu+mwL0ASXAdPO3JY7vXwPucrwPYWgsteZ7BXwpRXt/ALxuvh5t/v7cgOfW7vhOAac53j8B/Nh8/X+AH3n2tQlD6J2GIXSuAEoLPTb0X3H8ac1Ak09uUEqNVUrVKqX+XCl1TETKReQREWkVkcMYppmxph3dYodzJyJyjYisNk0+hzC0jPGOTTqUUlHz9THz/17H98cAy3ldC7xgmlIOYQiHKDApwTnUAj93bH8QEAwtxLe9PvwKuExEpgJfA7YppX4f8NyCUgv8ldVOc1/TgKlKqa3Ad4ElwD4RecZsi+YkRgsDTaH5K2AWxsp9NP2mGXFsY5fWFZFhwArgn4FJSqmxwKue7dNhB3CNKaSsv+HK8Gv4lfTdAdzn2X6EMsw/ce31QynVimFe+hMME9EvMzy3HsDpe5jsaeffe9pZrpR62mzDU0qpizGEhgJ+kqzNmqGPFgaaQlOBsVI/ZDpu/1uK7cuAYcB+ICIi1wBXZXH8h4G/t5zAIjLB9AtgHiMG1Hm2/zsRqTe3HyMiN2dw3F9i2O0vAp40P0v33DYAt4tIWESuxjABWTwK3C8i880orJEi8h9EpEJEZonIl0zhcxyj/2MZnINmCKGFgabQ/AwYARwAVgO/SbaxUqob+M/ArzGcvbcDK7M4/s/N378uIt1mG+abx+oB/h740DS1LFBKvYCxin7GNGs1AddkcNwVwDjgLaXU7gzP7S+A6zH8MHdg+Bcw97UOuAfDGd8JbAXuNL8eBjyI0ed7gInA32VwDpohhCilH26j0Wg0JztaM9BoNBqNFgYajUaj0cJAo9FoNGhhoNFoNBqMLMuiZPz48Wr69OmFboZGo9EMKhobGw8opSak+7uiFQbTp09n3bp1hW6GRqPRDCpEpDWT32kzkUaj0Wi0MNBoNBqNFgYajUajQQsDjUaj0aCFgUaj0WjQwkCj0Wg0DGFh0NjayUNvb6WxtbPQTdFoNJqip2jzDLKhsbWTOx5bTW8kRllJiCfvXkBDbWWhm6XRaDRFy5DUDFa3dNAbiRFT0BeJsbqlo9BN0mg0mqJmSAqDBXVVlJWECAuUloRYUFdV6CZpNBpNUTMkzUQNtZU8efcCVrd0sKCuSpuINBqNJgVDUhiAIRC0ENBoNJpgDEkzkUaj0WjSQwsDjUaj0WhhoNFoNJohLgx04plGo9EEY8g6kHXimUaj0QRnyGoGOvFMo9FogjNkhYFOPNNoNJrgDFkzUbLEs8bWTp2QptFoNA5yIgxE5Grg50AYeEwp9aDPNl8HlgAK+EQpdXsujp0Mv8Qz7UvQaDSaeLI2E4lIGHgIuAY4E7hNRM70bDMT+DvgIqVUPfDdbI+bKdqXoNFoNPHkQjM4H9iqlGoBEJFngEXAp45t7gEeUkp1Aiil9uXguElpbO1kxfp2BLhxbrW9+rd8CX2RmPYlaDQajUkuhMEpwA7H+3Zgvmeb0wFE5EMMU9ISpdRvvDsSkXuBewFqamoyblBjaye3LVtFb1QB8GxjO0/fs8A2G+kidhqNRuMmXw7kEmAmsBCoBt4TkbOUUoecGymllgHLAObNm6cyPdjqlg76ov0/t8xB1sSvi9hpNBqNm1yElu4EpjneV5ufOWkHViql+pRSnwObMYTDgLCgropwWOz32hyk0Wg0ycmFMFgLzBSRGSJSBtwKrPRs8yKGVoCIjMcwG7Xk4NgJsU5MgG9dOJ2G2kpdnkKj0WgSkLWZSCkVEZHvAL/F8Ac8rpRqFpGlwDql1Erzu6tE5FMgCvyNUmrAwnicZiIFLHvfkDtPrNrOib4Y4ZCwdNEcbp+fuV9Co9FohhI58RkopV4FXvV8ttjxWgF/af4NOJXlZYiAMt0GMWUIBKUM4RCJKX7w4kYALRA0Go2GIViOorG1k6WvNBPzuJ+VAul3IxBTsPilJm0y0mg0GoagMLCSyryUloS494t1hFwCQemkM41Go2EICgMrqcwx5yPA1xqqeeDa2fz4hrMIi/FZSUh0lFGO0M55jWZwM+SEgZVUdsWZk+zPFDBn6hgaWztp2tVFyFIPnHYjTcZY9Z5++vom7nhstRYIGs0gZMgJA4vPDxx1vX9n0z7ueGw1T69poy+qUEA0qmsT5QJd70mjGfwMuRLWja2d3LJsFZGo24O89/BxeiMxrE8FnYyWK3S9J41m8DPkhMHz69vjBIEAF9RVsWlvN30RI8/g5nnTXAXsNJmj6z1pNIOfIScMEhU0emLVdu68YDrNuw9zzZwpOr8gx+h6TxrN4GbICYM5U8fEfaaA430xOxN5TUsHzbu6tGag0Wg0JkPOgdzZ05vwu5gy/nqjiifXtHHbozryRaPRaGAICoMFdVWEA0aM9kZiPL++fWAbpNFoNIOAIScMGmorueeLdQm/98qJjB+aoNFoNEOIIScMGls7efR9/+rYInDfJXWUhQUBysLCTXOr89tAjUajKUKGnAN5xfp2ogmW+6Lg8IkIX5s3Le7ZyBqNRnMyM+SEQTJ3QQx4dt0OojFFWUmIG4eYVtDY2qlj/U8S9LUuHEO174ecMLhxbjXPrG0jGl+4FMB+6M2Jvhgr1rdndTGLaVBY9YF6IzHKSkI8efeCgrdpMFFM1zIV+loXjqHc90POZwD9D7VJug2GlpBpaGmxFWfT9YEyp7G1k9seXc0//3bToAg31te6cAzlvh9ywmB1S0fcg20SEYlm/jyDYhsUVn2gsBSu5tJgLWP9/Pp2u27VYAg3LoZrfbIylPt+yJmJKsvL0to+04tZbMXZCl0faDCrz961Qy7DjQfC/OS91gAPvb01o2MMJvNYMVDo+2wgyYkwEJGrgZ8DYeAxpdSDCba7CXgOOE8ptS4Xx/bStKsr8LbZ3PTFOCgKWR/IT1Mqhj4Jwk1zq3lu3Q76oorSHIYbD6SAtK51NscYzAK8kAzVOlxZCwMRCQMPAVcC7cBaEVmplPrUs10F8BfAmmyPmYwD3SfS2j4bJ3KxDYpCrvKKTVNKh4baSp6+94Kc910+BGQ2xxjMAlyTe3KhGZwPbFVKtQCIyDPAIuBTz3Y/An4C/E0OjpmQ8RXD0tp+697uAWpJfin0Kq8YNaV0GAjBng8Bmc0xBrMA1+SeXAiDU4AdjvftwHznBiIyF5imlPp3EUkoDETkXuBegJqazEpM3zS3muUftyVMPPNyIpIgBrVISbT6L4ZVXrFpSumQD9v+QPRNNsfI9LfazxCcwdRXA+5AFpEQ8D+BO1Ntq5RaBiwDmDdvXkYm/YbaSmaMH8nW/UdTbwyMGVGayWGA/F/oZKt/vcrLnHzY9geSbI6R7m8bWzu5bdkq27/y9L0XFP0k50c+7t1Ca+vpkgthsBOY5nhfbX5mUQHMAd4R4wH0k4GVIvKVgXIijxtZBgGFQdvBnoyOUYgLnWz1X2gzzWBaAXkpBq0qn2RzrVasb6fXVLt7oyrrxM10sNpdWV5GZ09vxmMtX/fuYBtXuRAGa4GZIjIDQwjcCtxufamU6gLGW+9F5B3grwdKEAB0H48E3vbq+skZHWN1Swcn+szY9L78XOhUq/9CmWm8N9fi6+qzulnzzWDQqjKZwL2/aWztZMX6dp5rbCcSzWwi9JZ7ad7ZRWNr54BfZ2uMWfecAOGQsHTRnLSfWpivSXowjCsnWQsDpVRERL4D/BYjtPRxpVSziCwF1imlVmZ7jHR48NXP+GxPMKfwiJIQD1w7O6PjVJaX2aGpMYLnN2SzKku1+i/U6tx5c/X2xVj8UhMxpYpCNQ7SJ8WuVfmtZMFYpScquOgnoJe+0mxPpuA/EaZqy41zq3m20UjSA9i4s4s7Hls94NfZGmNW2xUQiSkWv9TErMkVaR07X5N0ocdVuuTEZ6CUehV41fPZ4gTbLszFMRPxiw/8y1f7MS7NyCMn3nyGIPkNuVBPE63+C2mfdN5cIkI0puxs3kKqxun0yUBoVUEEUZA2erXQFevbeW7dDttc82xjO0/f4/6dd/X7WtNu12QqxGfQBmlLQ20lT9+zgKUvN/NJe1feTCDWGOvti+EM+Yip/ioCQSfdXEzSQRdegymoYshlIPelERy0+9CxjFVcr7oc5OFqA6meFtI+ad1cK9a3s3VvNx9vN8pRxFT6GeGZkKsIq1xqVkEFUZA2erXQA90n7IKL4P877+r3mjlTWLv9IH2RGOGQcPO8aXEaRTr99dnuw/brcHjgTSDOCbz7WB+PffC5rX1WlpfZJqSgpqNsJunB5hgOypATBukQU5knnVnqsnWzBSmHPZDqqXPf4ZCwKwtBlynPr2/nhEMah0j+TOpcYBWZs/rUuUJeUFdFSUjoiyrCIUna37m+wYNOrEHGRGdPLyExxmtIjFya0rDYmkGpOSE6S1L4rX5nTa5IKuyCjs/VLR1EzAJgAnytIT/PBXFO4FfWT7bPxak5RWKKH764MW3TUToM1MKr0EEYQ04YWDdNUDJNOrPU5XQuXq5siH6Dxrk6f3bdDp5a0+ZrPhgovDZdgJKwMQHnepA792cVmYP+InPOY0SVYV9OlXeSi4AAZ7uCTqxBxkRleRkhEcBYCd80t5qb5lbbPoP6qWNYsrKJ3qgiLPCjG87i9vk1cavfVKvhhtpKFl9Xz2tNu7lmzpSE23rPLR9PC7Sc35aPxNuekPRf46jPIi+XYzDVtc3U2V9obWPICYN0BAHAwZ4+1/un1rTZN0M6qma+pHqyQdNQW8nz69ttE4Lf5JjNcVNNWHF9L8KmPd0sfaU5Z4Pce/6XzJzg+t7ZhEfe3UbUbFQ0pnjk3W0s+9N5vvvNNCAgUbuevHtBYMGfbJJubO1k6SvNRGOKkGCfr/M333tho60lRBX8MAOnqvNYvZEYa7cfTLiPTBc1md4jVm6DdY7L1+0gJOKKiLp89iRe/3Sv/Rvx/D7XE+2Nc6tdgsnSyoC0j9XY2snP3txc8DDUIScMQkA6OcUzxo+0Xz+1po3vvbARgPe3HAAIFLaWzFTh3S7bQZlKRU1UgTMbYRWk3c0+DvRotN9xmatB7j3/8RXDKAuLb5G5vYePu37rfe88v9eadmOsvQ3tMl3zlt91+fZlp2V9Qzs1rqiCNz7dy3tb9rvCd731uGIxlVE/p2P+SNfmns3YX93S4fKRRKIKwQhSsNp536Wn8s7m/b5m21yadbznUT91jGuxc9Pc6rT9VM6Q2ZAUrjT2kBMG9h0dkMtmTbRfv9a02/Xda02744SB36SaylRhEWRQppq0U6moN82t5tdr24jEoCRkvM9WCKVqd2NrJ8+u2+H6jWA4Fp2Oy1wMcj8TxU1zq3377JbzavikfaPrvRe/m7EsRTv94vd3HjpGOCSoqMrYoep37a3ztdqnMJ7SZ4XvloQkbvEjkpnjfiB9WtlMyAvqqlw+kpKwEBIhGu1vZzKzbTrnlej+sz7fdegYx02/2Im++MWOgrT60CnsQ8BFp43nu1ecrn0GuSBdM9F/W9mvUl8zZ4qtEQBcM2eKa1unuhoC5k2vZOakCvZ7VmaJmhDE1uiXvOXNuEyloodCISQWIxQynl2U7cooVbudDkV3RyhmTa7Iaax1ovP32+/t82to6zjKb5r3cHX9ZFuwO294r6/jrFPGsPj6+qRmG6cWuOR6d/y+mOedDD9TZCKBbZ3vw+9u4w3TDKLADt/ti6q48RZTsOTl5oRmnkQTXqqxlY12mY2gaag1qsp6fQbW+yC/DzIGE10D5+fO4ymgamRZ3OJkztQx9vWF5M+a8PZLoQQBDEFhkC590X6V2roxE/kMnKn4MeDj7Z18vL2TkrBQEhaiHlOF9+ZxDsrK8jI7Ptq6+H7JW9ZNb61YnZOEE+fKJRI1IyuiMfv4JWEz0iiDVavTsVg1soyfvbmZ+imjqRhR6nKWOmPArciOTM0lXodhOo5Q5z6eWLWd3kiMJ1Zt50oz2/yWRz6yNaeli84iFBJi5nVt3n3YftKZ3zG8WuDytW1xyVDRJGYarymyreMoD1w7247Ecpo+nILunGljeeuzvcRUf/atNTb8SKZ5+pk0nWP125ed5tuX2dQkyjZ4wrrmzpIU1rVYsb7dTqpLpP16x4zzfAH73vFbNDnvSy8dR3td57VpTzc/fHEjMQWrth0gFAolzfbOVVBJLjjphQEEf9pZolVINKq4bX4Np4wdYV/Qp9a0+WbiWhfbbwXiTd6Kqf6bPZVpydqfM5rKFedvrVaDPCDag+VYPO4IG31/ywEEGFba7yz92Zub+WDLAbvNIUkezuncv/NmeGpNGz98caMdHZJpVJRfhNCGHYcw53IiMXjh9+0ox10eiSqeXNPG8rU7fOPV93m0wLKSkEsQ+tl8nefnNUUue7+FmqqRPLtuh91vfgLbu4KcNnYEW5LU3/ILNwV/kyakdnp6axL97XOfUDdhFBMqhvlmQPuRrp/BibeURsi8P7xJdUG0X+f9UhISMJ3RJSGhJBxymZ/AnfBmmeos6qeMdgkq57iNxICY0dfONvktEgspBCy0MAD7hti0pzupA/nGudUsX7eDiCdO0VIPnZFFzkHhzcRNZLaxVgkr1rdzoPsE72zeTySSeJKxcO7POddbcf6WGSfVqtWJnynFi3MV++3LTuO7V5zO2u0H6e2LETKTf6zJPZG25RVk06tG0nLgqGsVlqnTzy9CaJ/HibzjYI/vii8SU/zgRWMs3D6/xu4P74Lg9EkVPHDNbHu16q3J5DU93HnBdJcpUilDE00Vt+80F/2+rTOhIDjfNF16HZvWBO8VZvu6TwQyI3rPe+v+o3Zl4OXrdrB8AKuXWn3oXIzElEKk/76wfFO9fcZCKpnPxHW+UYW15IpEFbfOn+Za1IHR93deMJ1H3mtxCQIBKhxVj1e3dMSFMIdDAkrZwvmef1vH7/64D5WgXEshcw20MACeXNPGivXtzJpU4frc60BuqK1kuWm7tCI4/FZGK9a3uwaFeFbIyeynm/Z088yaNmJAWOC2+TXUTx3jmmS8A8abcBbD0FasOP9Ne7rtOHUReL15D5XlZb42dK+N1JrA/CZMwS2gnMJMgFmTK1JGaDlvzJjCt/R4UKefdzLu7Ol1xRM07erigroqPmnvj3xaUFfFb5r3mJOIsa1Ts1r8UhOAPbGWhITSsBCJKjtqZdOebla3dHDNnCkuE4tfyODhExHC0h8TL0Kckz1R3P6mPd2238APAU6bVMHff/UsHnp7q33c430xHn53G4/+6TwmekqwTKwYFsiMeOPcap7+uM1fcA5w9VJLw/OiFIRCwuLr6m3/0LL3W+yaReAfDVhZXoaIIEoRCkHU3HUMmDN1jO+CZZlHEIAx0VeWl7nGX0kInOumey6eQcWIUirLy+xcEAvvIrHQuQZaGJj0RWJMHD0c6J8ovA5kcNv3E0lv7yrq8jMmxtkv/ZJ7Gls7+cELG227e1QZK7cbzcxRaxu/AePUKN7etM9sSH+cfzSmECCi4JP2LjvKZtbkirj9eVeKzY7SA05qq8q595JT4/rAacv1Ctjla9tcE7Y1EflpHgDVY4fz55fNBPwdcYmigZ68e4FLMwB45uM2rpg9yRYQAsycVMH5M6psf8gHWw9w4Eh/WGlMKZcJIhJVnFU9hjmnjLEFgVPYPf7h53zrohl2v3pDBgV3AlxMwcefd7ji1hNNAF4TkxcFPNfYzk1zq+1ELGvyfuPTvTy1po36qWPsz0vD0h+CmcKM+EbznqTBGakcudk6n0Omj8RLNKbo7OnlqTVtLHu/xW6jt4idc8JesrLJ3pdSJA0pfmpNG//rrc2+EVsxpYx9KUXU9EF96Yz+fAcBDp+IUDGilOZdXa7wWIste7v5xi/WcM2cKXT29BY010ALA5NwSLj/0lO5bNbEpElnli8gGlN2HZRZkytczs76qWNcv1noCF/12j7XtHTQvKuLG83wSO+g+3z/EZfj7uZ50xJmyj7nqCYJxoByOji9Q3H52jauqp8cNwC9mkv9lNEu04ZF28Eelr7ijlrxChKvgP1092E27uyyo6WadnURjSXODNl56DiLX9qIQnxVa280kPM8dh065tpXTMFbn+2ltKTfLlxZXsaSl5sTCiMRGFEatgVWDEOYfrb7MDfOrY6boLfuO8L3XtjIVWdOcrVrYsUw/vPlpxvRVWvaXL95acMuwBiD9VPHJJwAvNFufkQdQQOjh5e6kiof/6CFNodZzGqb04wYiSp+9ubmuKiW3zTvSXjMMlND8nPKZpqI5UUlkUTdx/r4lzc2xwkryyTqPH5IxBX5FlNuU45TK3JqtU6MiDFDk3Cu9CMxw+xotxlYvnYHygwBDpsapfPYL5rX/v0tB7hk5nhbgy9EroEWBiYxjBXt0RMRdhzsoa0j3lzR2NrJ4pea7MEUiSm+98JG1wrs2cZ2Fp7uzoq1Kpo2tnbakSwWvabD8tl1O/jWRTPijjluZJltOumNKjbv7fbNlHU6Bi2UeWzLKaaUOyHPiqcPm9E0IbN+j9fc030i4jK3TK8qtycV7wqm+5gx+VgmJKeAHVEa5k0zIqa3L8YPX2ryXe15z8E4LdN5aWoc3rIPfg7ch9/d5ru/rzVU23Zhv35zEovBm5/tjVv5Wg92STRB7z18nJJQf2z8nsMnWPJyM0uur49LjLR6IFVJZssU8sy6HRzyZM5bhMMhuo/1ccsjq+LCfbuO9bknr6jir369gavrJ2POQcSAD7ceYO32g65J++r6yTz8nrsisNCvHQIup6wlWEpLQtzc0J+IdcI0WZ0zbWxgLWHF+vakiaSrWjp8x5ECO2qv3xQZv10spvhC9RhuOa/G1Z7la9vitg2HhItOreK9BEJ5r8cn48yAv+V8Y3H58ecH2brvSNxv3zODMsKm6Uv7DAqEFUViYQ185/MOEsXTOz/qjcTiMl0t/8JPXvuMRPNOb1Tx5h/3uT4TYKzHEdZ51K3GWoLG6xi02xaDW84zJr/uY32uG3rDjkO80bzHVl/7oopNe7rjIk9C4tYqJlYMY+ehY64Eq8bWTlcsPMD508fZjnHLCfveFiNLFEgpCKw+8DrtnjWd+FZo5OLr6lm+to1hJSFmTqqwTS1e+zhgZ4la5+gnMJwoEgdgCf026cc/aHH5O245r4Z3Nu1zlUiwol6cuxs/qozOnj67LyIxxdKXm31zHZ5a08ZjH3zun9Nhck71GB59v8W3FlPH0fis6u0dPXGTvJ+Qf+Da2ew5fNxeyYLRN60dhnboyrx15D70RmJs2dvt0kbe+HQvb366N3CIqjfD2suk0cMJhQ77jifLJOlX/tp5Hp+0d7Fpr1vLneTRak+pHMHC0yfQtLPLZy8Gp00YSWNPr9tnCC5f0DMfxwsZZ1uUUgNe4NGPUN6POIh4ccNO1/ugWZ0X1FURdvTsO5v389SaNtaapZ0TcazX/YQ2BbYTFAwbb92EUa5trO+a2g/57rMkLMwxzVbWCt8iElVx6r9l9nCuprwC7OPtnf32T2UIkDseWx3n3HxvywGeWtNGY2snD729FYA7L5jOtHHlVFeWx7X1tAkjmTDK3cdTK0dQEjZaHRbD32JNNr2RGI+8u40lK5v4pL2Lj7d38tSaNt5o3kNja2ecgLzqzElxJopUtm4wtI2wz4YVw/rXUiOHlRAytykJC7MmV3DI54aunzLaJQw6jvQiHgPeJ+1d3PboahpbjfHS2NrJzQ9/xPde2JhUEIAh4L2CQDDan05UsV9EztHeaNx2CmO1v6/7BGUlIcJmhI+Tgz5CSNGvXVnjwzpfJ42tnbz1WXKneXlZ2FcQhD2a7kUzx8fvwIGldVptue/SU/vHXgj2d5/g6Y/b4oSBdbolYeGGc6vj+v+86ZX2uHt+fXvKxNiY6tew84nWDIBTxg5n56H4ujXTxrknrKDS+t3N+13L2Ug0fkXoJRwyqk962+EUIDGlqHPUUgLjN0+taWP3Yf/V08JZE+0MWTOk2p4USktCceq/5TR3+g1iyj+rWmGs7i0Hqx+Pf9BCy/6jhgmHxHWjwiFhziljXCtPgJ2dRpkHwGi8h72Hj7tMHwpDq/vFh5+77LMh4OxpYzNSvc86xXAYe239y943+s27slYxY5L72Ef4txxwmx/7zWBurEgTgK8/8pEd8ZKKXs9MdP70Si6dNTFOK0xFJKbispg/P+AfzqowxvyS6/sz5hevbCJiRrTVTRjlGyUG8PvWTrt8Skjgx2bFVQu/cE3vsX/n0agt5kwd7QoPTeVziSlD64zG+n1TS78yxzZxvvHpXt/74IzJFRzri3LOtLH8w79/Gvf9waO9dju2BKiSbI3hmqqRaT/SMxu0MADGjxrmKwy+eq47xC+otPY+djMkktAJaxGNwfG++JWXd5s3Paukdzbt41iS3+3s7LEdzs6bKiyw5HojJK+mamSc03zTnm5mTapg0ujh1I0fmXAisWK817R0xE1E4A4VTTaf1VWVxwkCC6fd1TvBJlrZeyM3QuH+ZzwAdmTJW39MvOq0mHPKmLigADAmj6d8VP7SkpCvaUORuFieH1aWbVBB4CUkRrSUlSviNbmlwltna1x5acJt+yIxOnt6+fZlp9HY2kkI49qEMBYklq/Ii/NeiSn4gedZBAvqquLCNb2oBCrPjPEjWfSvH9hj2KvpexEMbVmZ57NifbttKk3Wd9Y5bO/o8f1+24GjNLZ2smlPt+8CIRHL17ZpYZBvnHHnTryVOFeZK7V0OX3SKLpPRFJul8gpZREOSZzAaDlwlCvOmJhQ0Hy6u9t3wowpw9/Q2NpJZ0+vK3rEHUXRxTnV8ROh1R4rxvuRd7fRetD/ZgiCd8UclE0JVlrhkKBiyhZA/ZnFbSBG6RARUqrsYJgHvu8TVQLxAnxkWZh/u2s+P3ntM9/tZ4wfmXC8eWne1ZXQFxSEcEh4ysymvvDUqrQEgYXzN17/lXe77mN9PPT2VqOsg8MP1dnTyxWeEtOJiCniSnFc94WpCRcKAJfPnuT7ff9nwfo7JBhjI2YEUwi4Et0yRZnntDrN+cOIxMsfOREGInI18HMgDDymlHrQ8/1fAncDEWA/8C2lVGsujj2QeG8eb5hkUD7d3Z1w0krF5Iph7DEnBEFRMbwU6F9djisvdWVB+pHIxPPrdTvsEFdnyJ83XHJDgskrFlO2QMlGEEDqh88k/F2C2XxuzViad3bR47mZnZFJQW3oySYxEfea0Xq13ScaDUjqfPSyv/tEIJ9GIiztKBJTKRcafgi46mwls90Ddoaus82WkFg4a2IgYSAQF96ZTBAAlA/LzZo26ogW6IsqmtO4VqnoPtYXKDTYoiQE9196as6OH4SsHcgiEgYeAq4BzgRuE5EzPZv9HpinlPoC8BzwP7I9bj6Y4zENZHNxojEYUZJ+d+9xrAwjMSOJxcnY8jIW1FVlNGlEosr2CZwwH7QO/sl2figMVdYq55EOJSEj8sXPMZsOiYRBY2tnnCAYEDwSpac3yh2PrebICX/TXVcajsFDPb2uuPV84zwzb1Z9su29m735x32B/W2nThjp8uukSrQD2NAW3PSSDp8mSLbMhGXvt/iGqydiTpJ8k4EiF9FE5wNblVItSqle4BlgkXMDpdTbSilrVK8GBv45eTnAzuQ1+dWq7Vnt73imxt8kjPcJn0wHs8q1nb3a2NrJrMkVSX/jJBqDtwKs+LxEYobGkalGYB8/we/TLWWeKSd8GnCiL0ZpAilXlsaC4EQkRmsaE8hAYC0QspHZx3ojgRynAFfMnuR6XzUydQTfjs5jKbfJhFE50jjAGI+PpOHA39DexYOv+psaB4pcCINTAOeTTdrNzxJxF/BaDo474HhXHK81Jc7CDEIGBUPjOObRDOZMHcMj727LyB5s/d660a3s1XRX+gcLEBNdzCgs85GbkMDo4clNek4uqKvy3U8+sZ4R7udAD0o0qhJG/Hjxmjz9ciO8hAYoQP5EMq91BqR7j6ZyeOeavOYZiMifAPOAf0rw/b0isk5E1u3fvz+fTfPFa5LpG4CVfbp0ejJP39m0jzfTWJk7p5aSsHCBWcPGynxcUJe+o9Evuetkp/t4vDloypjh7DkcfBXbcuAo40cVtm+tCfHF36deIEwePYwvzhwfpxUd7Onj8PHUARQQH7EXRDOQAdICk0XpDUVyIQx2AtMc76vNz1yIyBXA94GvKKV8QySUUsuUUvOUUvMmTJjgt0le8dZDycXKPlu8Tfho24G0nvns/L1SimVmtqoR7258m6hqZiJKwjp30YvfumHnoeMJfQl+NO/qSmhuyhfWo0LbAvgurMmz1JN1lqz2lBdvxN667QdT/qY7oKBJlxJv9lye2Zsgd2igyMVdvBaYKSIzRKQMuBVY6dxARM4FHsEQBMH0xSKgCOb+lBxNY3LxEo25besxBY+8u42G2sq0BkakCDSmoBRabCWKifdDSBy7ni8s/9HkAGGOXccivL/lQJzjPh3/jde30OWjYXnJ1u+UCCmwMFCQV79B1veGUioCfAf4LfAZ8GulVLOILBWRr5ib/RMwCnhWRDaIyMoEuysqvPdtgcdGQspLczfFWfH+ksYud3UFT6QqNIUWW+mYoaeMHZGy/MRAY/mPgmgGiUhHcezpi7kmwHHlhTOT+T1DId/k02+QE3e5UupV4FXPZ4sdr6/IxXHyTazAN2IQFEalSnI0cMeVl/Lgq5+llfVaDOazoUjQWlgDyQdmXHw4i5VQun7YFzfstAtE9vQOjAlosFAzLr6G10BRaK25qPGO4ULbEBNxIoeOrpmTKpLWrvdjoKI5TnY2Z5iomEt2dRkO73yOfecE6Fcc72TihnPzF4Wvb+M08IspLwb8agJlSv3UMZwzbWxavxlELoNBRaH9BdDv49h3JH/OTOcEOJj8UQPBCwGiuHKFFgYaF027uhiZw2QbzeDGMgGmERCUNc6aYLlc6AxG/hCwjlUu0He9xsXWvd20FcGKVFMc2I+tyOMx1/s81+BkJdeJb8nQwkDj4vMDR+MS2zSafJJpBVtNdmgzkcbF4eMRxgfI+tRoBopiyPQ/GdHCQOOiNxKjUgsDTQEpzpi9oY8WBhoXCvjjnsKHNGpOYrQ0KAhaGGjiOLnjNzSFpkwnrhQE3esajaao6M1nHKvGRgsDjUZTVGj/cWHQwkCj0RQVRVr1ZcijhYFGoykqBkF9yCGJFgYajUaj0cIgFQ+9vZVGnR6v0WiGOLocRQr+6bebCIeEHy2aU+imaDQazYChhUEAojHF917YWOhmaDQnDY2tnaz2PA9ZM7BoYaDRaIqOm/7PR4VuwkmH9hloNJqkTH/g3wvdBE0eyIkwEJGrRWSTiGwVkQd8vh8mIsvN79eIyPRcHFej0Wg0uSFrYSAiYeAh4BrgTOA2ETnTs9ldQKdS6jTgX4CfZHtcjUaj0eSOXGgG5wNblVItSqle4BlgkWebRcAvzdfPAZeLiM4z1Gg0miIhF8LgFGCH4327+ZnvNkqpCNAFVOXg2BqNRqPJAUXlQBaRe0VknYis279/f6Gbo9FoNCcNuRAGO4FpjvfV5me+24hICTAGiAsiVkotU0rNU0rNmzBhQg6aptFoNJog5EIYrAVmisgMESkDbgVWerZZCXzTfP014HdKKV2OSqPRaIqErJPOlFIREfkO8FsgDDyulGoWkaXAOqXUSuAXwK9EZCtwEENgaDQajaZIyEkGslLqVeBVz2eLHa+PAzfn4lgajUajyT1F5UDWaDQaTWHQwkCj0SRlVFk478e8/5K6vB/zZEcLA41Gk5SmpVfn/ZgPXDubv/nyrLwf92RGC4OAnFM9ptBN0GhOKhbU6bzUfKKFQQrGlpdywzlTefE7Fxe6KRrNSYE1KTXUVha0HScb+nkGKdiw+KpCN0GjOalQumpZQdCagUajKSp0Omph0MJAo9FoNFoYaDQajUYLA41GU2SU6FmpIOhu18RRXoAkI43GYvSI0kI34aRECwONi5IQDC/Vw0JTOOZM1Tk9hUDf9RoXkZi+GTWFRQcTFQYtDDRx7Oo6XugmaE5i6qeMLnQTTkq0MNDEcaw3UugmaExOxvyrCu0zKAhaGGhchIBTxo4odDM0JoU2mVjuo3wKJV2TqDBoYaBxUVoaYuakikI3Q1MkRGPG/9DJqKKcZGhhoHFRIsKNc6sL3QxNkWBpJvksEbG6pcN+rYVQ/tDCIAnDwu6ReDKMy2GlOsegWCiG8WbJAMljY5xmolih7WQnEVkJAxEZJyJviMgW839czVkROUdEVolIs4j8QURuyeaY+SQcdnfPyTAuaypH8Pz69kI3Q2NSaIFQZi6I6saPzNsx32jeY78u9PmfTGSrGTwAvKWUmgm8Zb730gP8qVKqHrga+JmIjM3yuHmhstwd1RAuwpGZ6zbtOXz8pBB6gwEFDC9wbQZrlX6sL5rxPtIdo79xCIORw7Smmi+yHWmLgF+ar38J3ODdQCm1WSm1xXy9C9gHTMjyuHmh3GMyGVGEJpRojmfuyvIynXRWJIwoCTGuYlhB27Dr0DEAuo71ZbyP4WneN1fXT874t5rMyVYYTFJK7TZf7wEmJdtYRM4HyoBtWR43L+wwbwSLWIHakYopozObMMI+V39ubSVNu7qybJHGj5KQpGX26IspThkzfMDaEwRLCEwbV57xPkakUetqwqgyHrh2tv2+0JpRoanIo2aUsqdF5E0RafL5W+TcTimlSGJWF5EpwK+AP1NK+c6rInKviKwTkXX79+9P81RyzzRPvH0xDsuwQE8kQxXe52rdOLda22nTIJEJxO/jC0+tYlIagrs0JAU32VnHr67MXBicW1PJzAnBfA77j/TS2Nppv596kue8dJ/I3DyXLinnN6XUFUqpOT5/LwF7zUnemuz3+e1DREYD/w58Xym1Osmxliml5iml5k2YUHhL0mTPQCwrwgJuo4aXcPR4ZgPGa2IaO6KEhtrKtENLi9GXki8ShVz6ffzelgNUlpcF3vfwshC7PdppvrFW5pkKpbDA/ZeeSveJ4FntztDSQgvDk4lsZ7eVwDfN198EXvJuICJlwAvAvymlnsvyeHnFWyPl6w3TCtSSxAiCytEtc/h4hMbWThpqK5k9OXji2YRRhbVrF5J0e76zpzfwtif6YhxJYxIdCCqGG0EUXWm0G4xQVMGIyNu0p5u9h08E+l1I3KGlnT2Z+yqKjeJbSrrJtn0PAleKyBbgCvM9IjJPRB4zt/k6cAlwp4hsMP/OyfK4eeFwgW/EIIQEhpfkxq4YU8aqrLG1kz/u6Q78uwNHexl2ktp2R5eX+H6eSFtKp9RCSITQAAT4z55cwdnVwYIEyszrevBoesJAKUNQ9kViLF/bFlhoXj57Eg21/RHqM/IY0jrQZLJky6fWndUdrJTqUEpdrpSaaZqTDpqfr1NK3W2+/n9KqVKl1DmOvw05aPuA83uH7RLg1+t25L0NqTIwvz5vGvVTc1PlUTAmq9UtHXEDN1kzYkoxf8a4nLRhsPG3X57NP3z1rLjr5DeH33DOVDs6JwgxoCYLx20i5tZWsvj6+kDbWpPxjAmjMjqWAnojwUMvLps1Men7wUxpBjN7qV+UxwBxci7nAuJV6cMFyI1PFmctwJX1k5lbE5frlxFVFWU01FbG2bWF5BmoVSPLBlUIoPMyZrrwPm3iKP7hq2dx+/waZk2uiBOWMc/8d8nM8fzs1nNpO9gT+Bgjy8LsOZz7cuL1U8cETixcay6I7r/01IyP1xcNLgy8kWzvbPJ1Q7rI120ZEjilMnOH9rg0/EUWx9MQpNmihUESvCr9QKqsZSUhXzv9qePdKzLnNgpYsb6dVQ6HWzYcNCM5Ont6XTfYvOmVlJWECIsxYMo8K5z/cuUs1nzu34Zi8y1PrhhmC/WQwHm1mQnSkWVhbp9fQ2NrJ7c9ujplvse2A0eB9MxE59ZUMm5k+hNIMkIYi5z93cFs+FY584bayoxNFnVpaBXeQ+wNIAwHqmRFSPpX8+GQ8OMbzmLh6ZkHtuwN2OdOBo2ZaKjjrd552gBV8zxt4iievmcBl/oMtG37j7jeVwx326gFMrbX+42z1S0dLKirosQhDX6/4xCLr6vnL6+axY+/ehZ/ccXp3HDOVKZXlXP/JXXcPr+Gw8f8/SuTM8yByJZEWty+IyfoM2fumIKx5WUZPYC9taOHxtZOnl/f7msGUZ7DW5NqOhVhF86ayNEchxaWlYZYUFcV2H59ycz+MVmZhmCqHVfOF2eO5x++ehb3XXpqwj52CudwKL5IYiF9BnXjR3LXRTOYXlXOPRfPYNbkCjbvDe5LcxKSzHwG+azNpIVBEryruJsGqJrnSDMp5+H3WuK+E48d40QkRllJCMHQJm6cW51xyelwWFyJZyVhY6JoqK1kocNWG4kqmnZ1saCuiqWvNPPPv93Eixt2sb2jh8c/2s6Dr37mO9BvOGcqZ1WPzaht2VBWEuL6L0zBLxLYe3MpYOmis9JO7jl0rI9bl61KuMIu9QgjK6s7aGipYKzgd3flLrS0euxwFl9XT0NtJRMDZjY7x1Y69YkunjmeX901n9vn19BQW8ny+y7kyjMncdqEkS6tc0/3CaLmRYnGFJs8gQsdaTquc8mR4xEefq+F7R09PPxeC7cuW8Xa7Z0pfxcOuU1XV505iXkZaqD5tExrYZCAfJo3LqirYkUCG+7t59e43t9yXg1P37OAv/7yLJ6+Z0HaeQHi+F8/ZbQdJy/A1xqq7UiOCZ7JQjC0ht5IzDXx90ZivLhhp++xCvFchPOnV/KtC6fz4oZd9AUwt3b19LL4paaMknv6oomDeks8jr9Pdx8GgoeWhkPCgroqxuTwqV/th46z5OVmnloTLLrHG+aZCiuc1FqkOGmoreT+S0/lq3Ork5qNHv/AvSC6Zs6UlMcdPdw/oitbjnie+NcXsPbL6RPdPqR3NmeeQDs8jeztbNHCIAEKd/ILPu9zRcWIUl/hc071GB64djb3X1LnMsk01Fby7ctOsyfuhtpKzpySeuI9p3oMw0oN2/+w0hC3nFdj+wKGlYZcms+Nc6vjNJAFdVV2qKGTRKvdLXu7AzkAc8nfXjObZnPiTURp2CgLURIWGtsOEclQFy8NCxMrhvmu3ry23gNHenlqTVu8c14M57KXuy+eQUNtZWBt1KuJJKI3EuOHLzXx9Jq2uO9OqRzhOhevqe1gipj/RWdP5eKZ41lyfb0rPBSgsbWTOx5bzU9f38T2A0cS7AGOewri3T6/htMmJvc5pJPIB1AeMHl0rEcQhwOWE/lsT7fLh9QbibEzw+TBSK6LjyVhyAmDXJ6Qd5AtqKsaEI1hQV2VUQbCs/OKEaU0tnbyxKrttB3s4YlV212p+k5SlQs4beIofnh9PYuvq+fC08az+Lp6bp9f43rvvIEbaitZcn296+ZuqK1k8XX1cX6AubWVnD89Xg1+ccOuhKupoP04eniJ774TsWJ9e8oHqi+cNZGLZ47nS7MmEstQEFSPHc4z915gC80gvNa0Oy5aRin4ePtBrjqzv6xXSPqfAxz0ecAT06hhFI35azS3n1/DbefX2NcmFlOuBVBfisiWV/6wmw+3HmDpK81x49TSKmPKiLRK1GN+5SfmpAidjnpDt1JwIuAEe6S3XzAJcMt507jizKTl1xKyuyuzqLB8WigGRr8qILkMxPKq9A21lUwaPYw9AbMpgzB7coU9CS86eyovbthlf1c1soyfvbnZvon6IjFWt3T4rrp+l2IFvm3fEW5btgpEiERjrN1+EIClrzRzoi/Gqm3GTX/7/Bp7n0tfaaY30r9t064unmtstycFAUodJoGbH/7I1ybvJByCW88zjvGkz+rUy4K6KsrLwkBqWy3Ac43tSSM+SsLCO5v2EYkqwmGhtCREJBIjFBLuvngGL/9hFzsPxd+4IelPpCoNCz+/ba59HZ68ewEPv7uNNz7da28/e8poPvbYl+unjGZ9W/x5nOgzTG8lISEaU4RE7IVIUDPN1DHD2dmZevUZFiMrOBKJxd0rleVlLKir4lnzGodNH5JFKhOXJWR6fcappVX2RWKICDGzM0MYfWqNkw3tXXYWPMBTa9pc94QfQcIvrUk1JNg+ilR84ZQxfLz9oNEXplYQ1NcSR4YL/JI8Og2GnDDIFX720sbWzpwKAoBZjlBRr439lT/stm8wod+O7OX59e0p1UkF9EaVXbyiLxLjtabd9kQUiSkWv9TELFM4OVdyvX0xFr/U5FpRhoCLZo7nu1ecTkNtJU+tafONfAhJvNP2xrnVcY7CRLzumGD9qB1XTtvBHrtdfZFYwnDECaPKOLem0t5nJKq46syJjK8YZk8WTkEQDkFDTSUnIjHGjCjl/S0HAGPFvGlPt8tM9+ifzuOpNW281rTbtnN7hUHFiFJO+EUeAW99ttfup0hMsWRl/7U4c0oFn+5O3F8hgdMnVdDY2pky+iQcDrHk+no6e3r5ZMchuy+skNNNe7qJWnkBnsJLyTSgc6rHsKHd0HpiKl6rtrTK15p2Uz9lNI9/+Dl9UUVJWFg4ayJvfLoXBUSjbkHyWtNu76HiSTFfloSMIIHOnl66j/X5Bmr4Mb+uir+44nQefncbv/vjPp7+uI2SkBF0kSh1YvTwEs6ZNpb3zLFit6EkxDnVY+LGRCpOpJGjkS1DzkyUK0I+Ejkbn4EI3H9JHWM9D8x5ccMuvv/CRhpbO+NuIOfkq6yd+JDOoqPU9BGUloS4Zs4Ul104pvrNAtZKLixGX8SU27RQUhKyBQHA4x9+7t82T+OiMXjk3W109vTmRAWeNHqYq10KwyFf4hOg7ReZojCE6dMft/GIZ5KYPHo469sO8Ul7F+9tOWAfJ6pg8UtNcaaQ2+fX2BE0b3s0NWtxcct57oAAC+8k3htVrFjfzlNr2pIKAuu3FcNKKCsJ2Te04D9HRszyELsOHWPhrIkMN31IZaUhKsvL+OGLG4maGlDEYyZSCaryCfCHdrf5y3v+lqb54dYDPP7h50TN8RTDMNtZvqzSErc24udAHlfe72MrDQvDE2TpCvDFmeNZft+F3D6/hm9fdlrKgnmlYSEsMNwMwd20p5u3PttLNKaMhVFUUT02sUn2S2dM5MNt8fPE1xqqMwpNz2do6ZDTDETSf3h3SIzojz5HpEwsquJU3XQdVU4WnT2VihGlLDx9Qpza++SaNp5dt4Ob501D6J/cw6bZwJ6Eov5mopvmVvNUAJPL7MkV/PirZ9m5BNZ+Fr/UREwpyhw3YkNtJU/evYDVLR1UlpexZGUTvQ7tI+bt5ASdHg5JnIN2XWunPQFYmkmm+NXMOXwiYtb0ce9ZKRhfMYyysNAXVbYD2NKAvOzuOp7wZrQEp/NaNLZ22n37uSc/ZNzIMtvv8v9Wb085wYMxmQVaGQOrWjpc16uzp5fK8jJ+8OJG+xwEY/L9pL2LT9q7KCvp1xIWmBFtTgVTpF8TbWztpOOovwPZr3a99/yfX99uX2vnOLLClq22O8clGAK2reMoj7zXYh+js6ePkBh1jO679FQeeXcb7T6mvdKwuBYsja2dPJukpEwIuHneNE4ZO8I+b+PecG/XmiSL/KUNu+L6IiRGaPGsyRX8em0b6SQVj8hjza8hJwwymVnqJoziWxfN4O1N+2y7b4z4yT94WCD8aNFZvPD7dnYc7GFBXRW/ad7Dyk92JSw81htVbN7b7Wr+dV+YQvmwEp5rbCcajcWtmiwaTAduKhV0rjkZQb+WY5VT8LsRrcmrsbUzrlsjUcXSl5tZbDqXv3VxHd97YWPcMa/7whSadnaxdf9R+7ODR3tZ+kozi6+rt1X3RLb6VNRNGMXnHUdttd2KgIr4qNfhsHDT3Gpumlttn++mPd2ERFAq3qmabFVW5rkWVrRMr5kHMqPKHZN/8KiR3b1pT7dLEAgw28cMZEVwbdrTbZunknHgaC/Pr2+nfuoYe3J/o3mP6xxOnTiKbfuOuExqTbu6OMV02npH5uVnTIwbL0EZN7KMh97eavfRs+t29JsYPaZDoX+s+XFl/WQe/2i7ndynMLSz3/1xH/ddeir3XXqqrzlRAZv2dNvXenVLR9Lw0DIzos5qx0Nvbw3sX3Ae00tMGb65J+9ewPL7LuQHL2zks4Bm0sl5fLjRkBMGmawyt+47wpKXm7m5odoeqCGJn/yNzFxSSnYrUeyTHYfoiype/mQXMVP1jltRO+iNxFyawSt/2M3y+y5wTV6Jbpi/vWY2tzzykW/bnI5e76T15N0Lkt6IYKzq/G6iT9q7uOOx1Tx59wLb8fyjV5o55gjw//zAUX7ytbPj2tYXidHZ08u3LzuNxtZOHv3A38wEhtPX8omUhAWlDBNaqWlv/t0fTbu3wJLr65k1uYIVjpWohdX1TiG39JVm02lrXZ+EzTCOAZxVPYZbzqtx9ZnTx9Jn9q0TpYxtvJPqF6rHsPj6em57dLU92Y0bWcbXzZwP6xjL17ZxtDfqmsyd7Ow8ZjvkQ2I4Hvs8JzOyLExpWOyVeTgsLF/bRjTWb1e3nLylJSHuc9QjWlBXRYjEARrnTa9kfdshuy83tHexrrWTshJjgrW0Q8FY0b+7eb99nPqpY2zB4TcOV7d0+Ar3qGnGWlBXRVjin8/RF1X80NSMSsPCty6akXB+ONu8Dl6n97DSEL3meHa24IZzptJxtJf6KaNdWgv4+8ms4I8FdVVpZTFfMTuz6KVM0D4Dk95IjP3dJ2w7uXflB8YkMn28f8xzWUn/uioSVTz+QQu9ZlKSZYNNxQV1Vb42fG9egR/OLE8ngjHhWOGh3kkryIovWdt7+2IuLeOLM92RPJNGD7fbdvv8/ryG0hLDRv3Q21t55N1tCVdgglGZ9fb5Ndwxv4alX5lDWPpt4u9s2mcLmZgyIp4sE5f3qWJRjw3cm0R3+exJSaM3KoaXEA4LG3d2xYVPOn0spSUhLvCMnZKwYXLx2sAtobLk+nr7Zjx4tJeH32uxTX+3z6/hpe9czE9u+kJc7L9fa2PKTIjzdOmM8SN5+t4LuPLMSZxdPYaZE0bZGlUkZtj5vQmNFg21ldx7SZ1rf+GQmWQWFr56bjXO57JFov1jTIHtzwiHhFPHj+TS0yfwheoxfOvC6Sx9pZmfvr6JOx5b7Rs6bfWt91ytgIrVLR2+taFC0n/v9UYVq1o67DwKKyDDyqPxCgLrnJ+8ewF/9eVZPPsfL+QfvnqWXWLjZ7eey6/umk9N1UjX/SHArefXcPv8Gq46c5JrTCRrayKChhbngiGnGZQ5Vj7pMr5iWELbpUXd+JFs3RefNFNeGqY30u+cSvcB4lZs+dJFc+zVTEmC6KFENNRWcs60sa4QR4Xh3Nu0t5lZkytcIX6JzE5ebppbzXPrdtAXVYRDMLem3yTlNactnDXRpbJbZS2sVa6l5VSWl9mhq8mkTTgkcap7xPSjRGKKlgNHXdsfMMtDNNRWUjOu3BX95Y0Q8/bF/ZeeymWzJtpOVC9HjkdcJhanz8DpY7Hs704WzproWulbUUeWRtXZ0xu36n6tabf9vXWMpYvmuPwAQNyq2KkZOAWC5UB/f8t+Qwh6znHf4eNJtUQrOVJhrCJvOa/Gtq+vWN/uEsrhkBBGUWpqBnOmjrEj0pzRPE27DqOUSho6bUUiOa9LSGDpojn2tl4PUfXY4Zw5dYxrLB49EaEkHLJNrpaZMpnG7eyPhtpK1/UAQ2NzojD8A84Qbe98EnSOSjcDPFuGnDDIRAwIhhppTTrJVuD3XXoqbzrCAC16PLUPyjwlncNmoHppSYjzp4/jDzu7XHHMzok5HA4Ri8Ts6CG/AZWIbh8hpOi/0b592WkpBZ6XhtpKnr73ApftdZ0Zxug1p1kVTxOZ2qz+fejtrYGcx3dfPAPANiNUlpfZfR9TRmSJk/GOOPCxHp/PPM+19U7gYEzAidpkfW6Z3ZLdqN5VrDM+/fb5NXGTip8J0qtFWBVl7/1iHY998Lnt9F98XT1Nu7qMEiMen4Fz4h1RGrYL68VUfBsTRTo522gHaAguIf3Iu9tc215+xkTOnjbWHmOrWzp8k92iMUVJyHheX7I+7ezpjXNuO8OyvYEjVaOGsXDWRN7ZvN82v7UcOEpJSLj1/BpudLQ9E6x70q9I5NJXmu2wYO98Yt1LK9a38/Inu+g+nji6Kd1AmGwZcsIgmqZWMLliGN+4cHpaE+OPbziL77+wMa5Gj0VJWFyx7iGBey6eQcWIUteK+OPtB+NWJw+9vZVINGbHXK9Y327fwE4bvx8PvvpZXHikUwg5I4XSvRG8v0mkXQTVPCrLy1IKAsGIDHL6OLzlGcZ4JnyrIBy4BQP4V511+g/ueGy1LaDsFbC5yo7RXxogFJK4jG2rlLV13kuur3f1Q6r6UZYp7eF3t7Hv8HFuOc8tMLy+nqWL5qRc1QI8+sHntgnuzc/2UhIS1+r4zgum07z7sEtLScSvVm13CeJfrdpu952z7EhpWLjv0lPjIvH8rrc3oinRufiFXf/szc1894rTjYcxeXa+caehDS+53sht+HDrAWKmr2nq2BEZCQJLADjv4ZKQxPkInObXZIEZv2/tTOpIVubvsxFa6TDkhEG6KRp7uk8EFgQW1k3zQ1PtdTJqWJgLTx3Pm5/1q6cxBU+s2m6vQu1kLocT1cI7mQoEykD2ZsCCMYkFnTTSwbuiTrbaTnRMK88gmUBQGGYfZ395nW9dSTQRp2Dwe+/E6T+wEuqumTPFXnFv2dvdH62lVJzG4yxl3RuJ0byri6fvSU8DAzjHsZr2a581Drzjxo/n17e7xqc1Gd5yfn/4ZDpjwltw7c3P9vLQ21vZdeiYy0F887xpcfv1Xu+zq8cw55QxgVfofuPlw60HWGsuqCzHuKXtOPvpu1eczlpHJvGuQ8ds34QzFDdZfziFsdBvlotElSv9J+Twh/kFaji1/CAP/ckmnD1dhpwwSDXB+PHwu9t49E/npfUbSyD842uf0n28v4bJkRNRfvfHeDOSNZF7zRzdx/pckRR+posV69sTrrStQXrcp0Tnl86YaKuruSaZduH3ndfUFVQzcEb4xFT8A9Ktkt5+/ZPKZOXEK4S/e8XpQH+5DmdbvWUaIH7MqQT9kIhEUV6J2hfEluxtk2XeuinFBJzILDl9XDkberrs90d7o/z09U1x2oZfcT3v9fZqPslobO1k56FjdukQTJOQNeE37eoyTarGxBwOCbFYvzZs3VMr1rfzXKORYPjsuh0gYju4Q2bQSCLN2ymMXX0aEjsZLwRcdNp4W1vxC9SwrnFJSAKVtggazp4LshIGIjIOWA5MB7YDX1dKxYcDGNuOBj4FXlRKfSeb4yYjlCRVPBH7Mni0YGNrZ1wiloX3+E4b8+qWDnuSE3DZfp1hns4BmWylbQ06P978bC/vbdmf1LSUSxJNIo2tndy2bJWd6PX0vRe4VnqCcS/7BRRNNEtFWCv2GR4H/i3nJc6TSGcC9dNoHnp7a1zJbsFd6tvC6WS3/E/p4Dd5WMew+jWIw9OJVxO68sxJceYbL8mEkjeyxarVFETbaHYU6AthRH0lCyf1a09JSLhtfg31U8ew9JVml/Zs1cuKKZCYivMLWH4LK8rJuG/dWlMizRv6x5JzYSAYC673t+x3LSJsR7Fn7LmsAlHlmyjnZTBpBg8AbymlHhSRB8z3f5tg2x8B72V5vJSMLCvhcBKnjB/eMMAg+CWwWJOaZWOOmtE3t5znHpjDSt0Fu1INxGQrTGuQ+mkGqfabS5w297BpnrJWfivWt9tC0yqzcNPcarsfrMiOpl1dPPNxf40jhREB4tzOivjxRuMk6rd0nOXefrb6trfPKOpmrR79Jnqvkz1IfzuFZyLBlUpjSIZzVSnA2dPGpvxtMqF0zZwprgS40rAQjSpExBVB43eezszfUFh4rrGdSDT1Oa1u6eivnxU17P3eREmA5Wt32KaqmML3WdPWE/ycCzinb8gZ7pzI/GlpF5YWdP+lp3L/pafGXfdEY88rUFIxaDQDYBGw0Hz9S+AdfISBiDQAk4DfAOnZY9LkS2dMTFnl0EsmsbwL6qpcCTxgDSojueXxj7YTRREKheJWKM6CXU+s2p6W6u/FGnR3/XIthzwmFGuA5yM8zXXTeoreeaNWhMQ3y+8+2+sKB92w41Dcdn4hfonIxFnu/K23xEPQMMRU+E3yfv2RbHJOhXNVqQi2ykymTVmlIX7TvIer6ydTUzXSLmXijKDxsrqlw+VTqJ8ymo07uwKdk9O85Axj9vb10kVzXMUULX+CU9A01FZy87xp9sN9nL4hq3yH5Rj2E1LWMW+aW82K9e0pa2v5RRI5BUoijd57/vkiW2EwSSllFU/ZgzHhuxCREPBT4E+AK7I8XkrKh6V/Sp/sOOQqmxsEayW49OVmPnEU6YrFFM27D7sigrwqv7M0dLqqf6K2eGsezZwwksljR3DNnCl5MREtMBPm+ldn/QlzN86ttssiOyNr/CbPG845xRUOeXX95Ljt0gm1zZZshEky/CZ5v8TCTHwFFun4TCySaVPWszV6IzGeWLWdm+ZWB9JsnRpWKCRcUFfFpr3dgc4p6DnMmlzB18+bRvPOrqSC5kZzIvcz61hmwSBCygoYsHwPQbQcq38tgfKNX6ympze5QCgqzUBE3gQm+3z1fecbpZQSET/t58+BV5VS7d7n+foc617gXoCammArPy9bM3hg9eufZm5brz9ljDn5G6duVQNd65M/AJlFhQRh5qQKl/P8844eth04ytrtBwfEieydkBtqK+3VmV/Ru0SRNd79XFk/2Q6HDIeEK+snxx03U7NJtueYSyHkN8n77T9dU1eqYwQhkQD0jl1F4hBj7/4WX1dvj40nVm0PvAgKcg5On1Q4hMuZ7VdFwNufzpDRIOfj6gfT96BIT3Ozgih6ehP7DYQiSzpTSiVczYvIXhGZopTaLSJTAL8nrFwAfFFE/hwYBZSJyBGl1AM+x1oGLAOYN29eRikXflUsg5DOhWxs7bRVvUjUcGxddeYkJlQMs01CVn0cr/jLZqWXDKuOiuWLsB800pd7n0GiCTlI0btU+zFixi2nQXxl0GzMJunw1Jo2l2BbfF19UhNCungnJSChkMtUO3GaJHOhIXrHrrfoX7L9d/b0urSIoIugIMLQ6ZOKxODKMyYkDNG19unU1J39HkRIOfshHBIQSVpIMhH1U8ckLc6Y55yzrM1EK4FvAg+a/1/ybqCUusN6LSJ3AvP8BEGuGDeyDPYfTb2hg3Rs697kJDCiKc6eNjZucFuq5Ir17a5IIctumMrmmA7Om8b5AA+/6qt+55TOyjPZhJzOxOW3n1TCckFdlV1u3C/EMxc0tnay+KUm2+TVGzEeBJRrIeTsq3RMFOmcx5KVTfRFFWtaOrLWEBNNzEH2mc0iKNWY8t5HEyuGBda2M9HU/QR5Jpqbt3RLovblw8wL2QuDB4Ffi8hdQCvwdQARmQfcr5S6O8v9p423BEEqrMcwBk1+8RY3M/YRX0Mo1QrWT1Bki3XTPPT21sC24kzMLrnSbvz243XaWvHZrjY5NIeBwCqdYBESSWr6ywUDoTF6o7icJcczJRstJVNzVyoS+aSCkCtTWibnk8ofMKhqEymlOoDLfT5fB8QJAqXUE8AT2RwzFd4SBKlQirTS073hhoDvE8iSDbKBNHU0tnayYcchRIQQbtu9H5m0JVc3dqqVpp+QsiJTrPj2gVg5WSY3y+FphckmMoHlgnT7NIg25x2VzpLj+VptOsmVM97PX5VJtrfVpoESUqlYUFflW+7a4itnT81re4ZcBvJNc6tZ/nFb4DKxIUm/MuiTdy/gZ29u5gPzUYgRnyeQJRtkyQRFNk5Ky5Fm16v3qaHjJddOxnQJ6qy0+negfC7eNiUSUrm6OZ3PS3bmSgT1WQXR5qwVszOEMV95JwNFsudxZLMoSfbbgYxec+Y9VJaXcKinvzLuq017+EaaUY5ZtSUvR8kjDbWV3PPFusAPvb774hlpd3ZDbaUrASem/O3yiQZZoskm20gZbyJcLBZfQydoW7LBb6JLhfOGA9h56JhvVMhArKD9yOXE7+WpNW32U+GsMRS0nyC4NmetmL2JUvk0PeSafAUQWAxk9Joz/yIsMOeUsXzgSOrLt+AecsIA4JU/BE86S/WA7EQ0OdLrwahrno6Dzm+yyXagexPhgt74uV7xpjvReUsOWHHbicoN53oFnW+8zzX2PrcgFemW2WiorQwc9VPs5EMzdDKQwsd7LtfMmcKalo60799cMeSEQWNrZ6CaHxaZuiCth6hY5MIem+1Ab6jtr5UukHXN9kzIZKJLFLcdiSrfsgJBCVLvpxCTo7esg/e5BanIRJsbSE0nn+RKkw16/QdS+PidizMkPd/375ATBlb0SRDCIdIuKGbh56jONqY/FwO90Dd9JhOdX9x2JGI46P3KCgQl0Y1caI3BEo7pmtIsnGa4oTDBp0u2YzxZHS2/Yw2kg9l5LpaASlVVdqAYcsJgQZ3/w7H9CKXIiE6GX238IDH9qSj0ZJ4tmUx03hsO4GdvbrYfSJKpep7oRg6q+g+k9uD3tLMgZOtvGKqkc62S1dHyYyDvSSuB9UD3Cd7ZvD9wWYuBYMgJg4baSi6fPSllMgcYZohMV/Jvb4pPthbyW0vESyFNH04ymei8N5zzgSTZqOd+N3LQEgdD0d8wFEl0rRLdD8nqaOW73bc88pHrUadQuIivIScMvI/gS0YoJBmv5P2egRC0MuRAUKyTVzbcOLd6QGynQVT/fEetBCVbf8NQxO9aQfLyHonqaOWTFevb4wSBkH/HscWQEwbe8EoBZk+p4NPd7gJ2xgO0k5feTcYt59XwSftG12chCqcZOG+I3r6Y/XzYYpjA0sUr2NLJKA1KKtU/31ErQcnW31BIBkpz9btWqYT5QCcRBsFrpA4J3OYTPZcvhpwwsLL6LHmgIE4QWF/EyHzVZ9V2f3HDTvYf6Y176Hy+8WZGZ+N4LSSNrZ387M3NBV+VD7TjMBsy9Tfkgkwn9IHUXBNdq2IU5k6cSYHhkPCjJI7sfDDkhEFQn4Hz6UaZDBRnbfeQGKWsbzmvpmCThnVD5MLxWgiclWCdz6Ut5I082J35uSabCX2gzW7ea5VKmBeDWdVKCiyWBceQEwYA9116Km/9cZ+r2JgXof/h1ZlcBOfgjin4Q3sXm/ZmZnLKFQ21lTlzvOYTv0qwzoeLF/om0RhkM6EXwuyWTJgXi0+omBYcQ1IYNNRW8iPPY/C8hEOSVZx2ZXmZq8CU9+EWhYrsKWbzRiK8lWAFKCsNaUFQZGRbhrqYxmXQcymWCL18IGqAygBny7x589S6deuy2odlf3ZGX4DhPBbISj186O2t/PNvN7kmsGGlxv4gcSSDJh5LM7CSzm6eNy1nTrST6WbOB0OpP1OdS75MSbnuUxFpVEql/az5IakZQH8HXzNnCqu2HXCHcGXpPIb+OkDWo/ZucTwTYSAeVDKUGahVYzHYhYcaxWTWyJZU55IPU1IxjdEhKQy8HfylM9wO5VBIchP9IwIoQqGQayVbrGGJxcxATDLFYhfWDE7ycR8X0xgdksLA28HjK4ZR5qjmKeJeyWd6jEjUsHNHPc8zKDb76MmKFsqabMjHfVxMY3RICgNvB99kZrI+uaYNgFhUpfV0syDH8F7EoaROD1a0UNZky0Dfx8U0RoekMID4Ugab9vQnnsWAT3YcojGLpwgV00XUJEYLZU2xUyxjNCthICLjgOXAdGA78HWlVKfPdjXAY8A0jCjMa5VS27M5diISlTLo7Ol1PW/0jU/38t6W/Vk5bIrlIloMpUgPjUaTX0JZ/v4B4C2l1EzgLfO9H/8G/JNSajZwPhCsklwGJCpaZZl1rHogzryAoYAlBH/6+ibueGw1ja1xMlmj0WgSkq0wWAT80nz9S+AG7wYiciZQopR6A0ApdUQplfnjq1JgTfphTykDy6xz2/wa3+8HO4mEoEaj0QQhW5/BJKWUVWB9DzDJZ5vTgUMi8jwwA3gTeEApFc3y2L44bfmV5WX2pGiZdBpqh87zYJ0UU1SCRqMZfKQUBiLyJjDZ56vvO98opZSI+KUzlwBfBM4F2jB8DHcCv/A51r3AvQA1NZlX77Mm+GT1zIeKELAYSg5t7fvQaPJPSmGglLoi0XcisldEpiildovIFPx9Ae3ABqVUi/mbF4EF+AgDpdQyYBkY5SgCnUEChlJ9/6AMBSFXTBmZGs3JRLY+g5XAN83X3wRe8tlmLTBWRCaY778EfJrlcVNimU1CYNf3147V4kf7PjSawpCtMHgQuFJEtgBXmO8RkXki8hiA6Rv4a+AtEdmIUdPt0SyPmxLLbHLRzPF2SKmeXIqfRAEAGo1mYBnSVUvBMDvctmwVfVFFaVh4+t4LtNmhyNE+A40mc3TV0mSYBeWM/5piZyj4PjSawUa2ZqKix6+gnEaj0WjcDHlhoG3QGo1Gk5ohbyYaSvH3Go1GM1AMeWEA2gat0Wg0qRjyZiLN4KKxtZOH3t6q80E0mjxzUmgGmsGBzj7WaArHSaMZ6BVn8aOzjzWawnFSaAZ6xTk40JVXNZrCcVIIA78VpxYGxYeO/NJoCsdJIQz0inPwoCO/NJrCcFIIA73i1Gg0muScFMIA9IpTo9FoknHSRBNpNBqNJjFaGGg0Go1GCwONRqPRaGGg0Wg0GrQw0Gg0Gg1aGGg0Go2GIn4GsojsB1qz2MV44ECOmpMvdJvzg25zftBtzg/eNtcqpSaku5OiFQbZIiLrMnkodCHRbc4Pus35Qbc5P+SqzdpMpNFoNBotDDQajUYztIXBskI3IAN0m/ODbnN+0G3ODzlp85D1GWg0Go0mOENZM9BoNBpNQLQw0Gg0Gs3gEwYicrWIbBKRrSLygM/3w0Rkufn9GhGZ7vju78zPN4nIl4uozX8pIp+KyB9E5C0RqXV8FxWRDebfyiJq850ist/Rtrsd331TRLaYf98sojb/i6O9m0XkkOO7QvXz4yKyT0SaEnwvIvK/zHP6g4jMdXxXqH5O1eY7zLZuFJGPRORsx3fbzc83iMi6ImrzQhHpcoyBxY7vko6rArb5bxztbTLH8Djzu/T7WSk1aP6AMLANqAPKgE+AMz3b/DnwsPn6VmC5+fpMc/thwAxzP+EiafNlQLn5+j9abTbfHynSfr4T+Fef344DWsz/lebrymJos2f7/wQ8Xsh+No97CTAXaErw/bXAa4AAC4A1hezngG2+0GoLcI3VZvP9dmB8EfbzQuCVbMdVPtvs2fZ64HfZ9PNg0wzOB7YqpVqUUr3AM8AizzaLgF+ar58DLhcRMT9/Ril1Qin1ObDV3F/B26yUelsp1WO+XQ1U56FdyQjSz4n4MvCGUuqgUqoTeAO4eoDa6STdNt8GPJ2HdiVFKfUecDDJJouAf1MGq4GxIjKFwvVzyjYrpT4y2wTFMZ6D9HMisrkXsiLNNmc9ngebMDgF2OF4325+5ruNUioCdAFVAX87EKR73LswVoIWw0VknYisFpEbBqB9fgRt802mOeA5EZmW5m9zTeDjmma4GcDvHB8Xop+DkOi8CtXP6eIdzwp4XUQaReTeArUpEReIyCci8pqI1JufFX0/i0g5xkJghePjtPv5pHns5WBARP4EmAdc6vi4Vim1U0TqgN+JyEal1LbCtNDFy8DTSqkTInIfhjb2pQK3KSi3As8ppaKOz4q1nwctInIZhjC42PHxxWY/TwTeEJE/mivgQrMeYwwcEZFrgReBmYVtUmCuBz5USjm1iLT7ebBpBjuBaY731eZnvtuISAkwBugI+NuBINBxReQK4PvAV5RSJ6zPlVI7zf8twDvAuQPZWJOUbVZKdTja+RjQEPS3A0Q6x70Vj0pdoH4OQqLzKlQ/B0JEvoAxLhYppTqszx39vA94gfyYalOilDqslDpivn4VKBWR8RR5P5skG8/B+zkfjpAcOlRKMBxlM+h35tR7tvk2bgfyr83X9bgdyC3kx4EcpM3nYjipZno+rwSGma/HA1vIg/MqYJunOF5/FVhtvh4HfG62vdJ8Pa4Y2mxudwaGc00K3c+O408nsWPzP+B2IH9cyH4O2OYaDJ/chZ7PRwIVjtcfAVcXSZsnW2MCY+JsM/s80LgqRJvN78dg+BVGZtvPeTmhHHfOtcBmc/L8vvnZUowVNcBw4FlzMH4M1Dl++33zd5uAa4qozW8Ce4EN5t9K8/MLgY3mANwI3FVEbf5HoNls29vAGY7ffsvs/63AnxVLm833S4AHPb8rZD8/DewG+jDs0XcB9wP3m98L8JB5ThuBeUXQz6na/BjQ6RjP68zP68w+/sQcO98vojZ/xzGeV+MQZH7jqhjabG5zJ0ZgjPN3GfWzLkeh0Wg0mkHnM9BoNBrNAKCFgUaj0Wi0MNBoNBqNFgYajUajQQsDjUajKRpSFafz2f7rYhS5bBaRp7I5thYGmqLAUTW0SUSeNVPsC92mhSJyYZb7mCoiz6X5mztF5F/N1/eLyJ9m0wbNoOIJAtaYEpGZwN8BFyml6oHvZnNgLQw0xcIxpdQ5Sqk5QC9GPHVKzCzzgWIhRg5CYLztUUrtUkp9LdMGKKUeVkr9W6a/1wwulE9xOhE5VUR+Y9YZel9EzjC/ugd4SJlFAZWRbZwxWhhoipH3gdNE5HoxnknxexF5U0QmAYjIEhH5lYh8CPxKRKabN8l68+9Cc7uFIvKuiLwkIi0i8qBZa/9js9b7qeZ2E0RkhYisNf8uEuM5GPcD/8XUWL7ot51fe5wnYratyXx9p4g8b97YW0Tkfzi2+zMxnrHwMXCR4/MlIvLX5uvTzH74xDxPq/1/Y7bnDyLy383PRorIv5vbNonILQNypTT5YBnwn5RSDcBfA//b/Px04HQR+dAssJhV1VpdqE5TVJgr62uA3wAfAAuUUkqMh+f8V+CvzE3PxCjGdcw0KV2plDpuqs5PYxT8AzgbmI2x2moBHlNKnS8if4HxTIPvAj8H/kUp9YGI1AC/VUrNFpGHMZ5z8M9m257ybmfu29WeFKd4Dkb5kRPAJhH5/4AI8N8x6jt1YWR0/97nt09iZE+/ICLDgZCIXIVRUO18jGzllSJyCTAB2KWU+g9m28ekaJemCBGRURja6bMiYn08zPxfgnHtF2LUTHpPRM5SSh3K5FhaGGiKhREissF8/T7wC2AWsFyM+v1lGPV3LFY6Jt5S4F9F5BwgirFislirlNoNICLbgNfNzzdiPFQI4ArgTMfNNtq8Cb0k225lAEEA8JZSqstsz6dALUY9pHeUUvvNz5d7zgERqQBOUUq9AKCUOm5+fhVwFf3CYxTGBPE+8FMR+QnGQ1veD9A2TfERAg4ppc7x+a4d48FBfcDnIrIZ49qvzeRAWhhoioVj3gFvrpr/p1JqpYgsxKgrZHHU8fq/YNR2Ohvj5jnu+O6E43XM8T5G//gPYWggzt/hmPQJsN1R78YJcLYnSvb3oAD/qJR6JO4L4xGZ1wI/FpG3lFJLszyWJs8opQ6LyOcicrNS6lkxBtsXlFKfYJTZvg34v2JUWD0dQ/vNCO0z0BQzY+gvF5zsGb9jgN1KqRjwDYxHFabD6xgmIwBMDQOgG6gIsF22rAEuFZEqESkFbvZuoJTqBtrFfPCOGM/6LscwVX3L0lBE5BQRmSgiU4EepdT/A/4J4/GJmiJHRJ4GVgGzRKRdRO4C7gDuEhGr8Jz1pLXfAh2mhvk28DfKUS48XbRmoClmlmDYSjsxnko2I8F2/xtYIUYI5m8Ivkq3+M/AQyLyB4x74j0M5/HLwHMisghDCCTaLiuUUrtFZAnGJHAIo9KnH98AHhGRpRiVLG9WSr0uIrOBVaaGcgT4E+A04J9EJGZu+x+zbadm4FFK3ZbgqzjnsDKqjP6l+Zc1umqpRqPRaLSZSKPRaDRaGGg0Go0GLQw0Go1GgxYGGo1Go0ELA41Go9GghYFGo9Fo0MJAo9FoNMD/D0dZZNjdBFWYAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(torch.arange(0,len(conc)), conc, '.')\n",
+ "plt.title('Parameter Values') \n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"Parameter_Values.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAv/0lEQVR4nO3de5wcVZ338c93JgkQCJCEAQMhJIFwCa4GGCGIKyiCwIKwihpkJSgssouurOuusOsjiPosuiKuNyACAj5cFdGIKETkIgiBDIRLApEhJCQhhJAbIYEkk/k9f9TpSaUz092TTM90Zr7v16teU3XqVNWp6pr+dZ1TdUoRgZmZWSl1PV0AMzOrfQ4WZmZWloOFmZmV5WBhZmZlOViYmVlZDhZmZlaWg0UfIGmkpJDUr4K8Z0p6qDvKVaYcMyQd1dPlsL5F0t9KmtXT5ahFDhY1RtIcSWsl7VKU/mT6wh/ZQ0XLB503c8NT1dhWRBwYEfd3xbokHSWptajcv+2KdW9NJF0saV3a/+WS/iLp8J4uV1dJn/P8Ti4TkvYpTEfEnyNiv64v3dbPwaI2vQScVpiQ9DfAwJ4rziZ2jogd0vDurlxxJVc/m7n8K7ky7xARJ3X1trcSt0bEDsAuwH3AL7p6A8r4u6WX8Qdam34OnJGbngjckM8gaSdJN0haLGmupK8W/kEl1Uv6rqTXJc0G/q6dZa+RtFDSAknflFS/JQWWtLukyZKWSmqW9I+5eddJ+mZueqNfgOlq6iuSngZWSeqX0j6U5tdJukDSi5KWSLpN0pA0r3C1c5akl4E/daLMZ0p6WNLlkpYAF0vaJh27lyUtknSlpO1yy/x7Om6vSPps/peppPslnV20/ody0/tLmpKO0SxJnyg6Rj+W9DtJKyVNlbR3bv6BuWUXSfpPSe+QtFrS0Fy+g9M50b/UvkdEC3AjsIekhrRsh+dF7lj9SNIKSc9LOjq33fslfUvSw8BqYHSZ/T1B0sy0rwskfTk370RJ07Xh6udduXlzJH1Z0tOpHLdK2lbS9sDvgd214epxd0mHSnokrWthKv+AtK4H02qfSvk/2c65eUDat+XKqkY/Uuln1ts4WNSmR4Ed04laD0wA/l9Rnh8COwGjgSPJgstn0rx/BE4EDgIagVOLlr0OaAH2SXmOBc5my9wCzAd2T9v7v5I+2InlTyMLajunL7K8LwCnkO3n7sAy4MdFeY4EDgA+3MlyHwbMBnYDvgVcCuwLjCM7PnsAXwOQdBzwZeAYYAzwoUo3kr7MpgA3AbuSfaY/kTQ2l20C8HVgMNCcyoOkQcAfgT+Q7f8+wL0R8SpwP/CJ3Do+DdwSEevKlGcA2TmzhOx4Qvnz4jDgRbKrkouAXxWCdm7b5wCDgMVl9vca4HMRMQh4JynISzoIuBb4HDAUuAqYLGmb3HY+ARwHjALeBZwZEauA49n4CvIVYD3wr6nMhwNHA/8MEBHvT+t7d8p/a9Ex6g/8Frgn7cMXgBsl5aup2v3MeqWI8FBDAzCH7Evoq8B/k/1TTAH6AQGMBOqBtcDY3HKfA+5P438Czs3NOzYt24/sS3ENsF1u/mnAfWn8TOChDso2Mq1neW74MrAn2T/loFze/wauS+PXAd/MzTsKmF+0z59t7zik8eeAo3PzhgHr0v4UyjS6xDE9CmgtKvcn0r6+nMsnYBWwdy7tcOClNH4tcGlu3r5p2/uk6fuBs3Pz244l8Engz0Xlugq4KHeMrs7NOwF4Pvf5PNnBvn0SeDiN1wOvAod2kPfidN4sT5/XEuCoNK+S8+IVQLn5jwGfzu37JUXlKrW/L5OdszsW5bkC+EZR2izgyNx58Q+5ed8BrmzvvOrgGJwP3JGbbvv8itcB/G06nnW5+TcDF5f7zHrj0BfqaLdWPwceJPv1dEPRvF2A/sDcXNpcsl/BkP36nFc0r2CvtOxCSYW0uqL85ewSuV//kg4DlkbEyqJtNnZinaW2vxdwh6TWXNp6si+4SpaH7Bfn8HyCpDOLlmsgaxtqyh0bkX0JQ3Zcm3L588e1nL2AwyQtz6X1I/ucC17Nja8Gdkjje5L9om/Pb4ArJY0C9gNWRMRjJcpxW0T8g7IbKG4HDiH7oq/kvFgQ6VsxmUt2TAryecvt78fIfhBdqqz68YKIeCQtN1HSF3LLDSjaTvFxys/biKR9ge+RnYsDUxmaOspfZHdgXkTkz7v8/1l7ZdmBXsrBokZFxFxJL5H9WjmraPbrZL+s9wJmprQRwII0vpDsC4bcvIJ5ZL8gN/rC30KvAEMkDcoFjHx5VrFxA/072llHqe6P55FdeTxcPEMb7g7b3O6T88u9DrwFHBgRC9rJW+q4Qun9nAc8EBHHbEYZ55FVd2wiIt6WdBvwD8D+bBx8OhQRr0s6B5gm6SYqOy/2kKRcwBgBTM6vtqjMHe5vRDwOnJyqej4P3EZ2bOcB34qIzanOae8cuAJ4EjgtIlZKOp9Nq2U78gqwp6S6XMAYAfx1M8q21XObRW07C/hgZPWxbSJiPdk/17ckDZK0F/AlNrRr3Ab8i6ThkgYDF+SWXUhWB3uZpB2VNR7vLenIzS1kRMwD/gL8d2psfFcqe6E804ETJA2R9A6yqoDOuJJsX/cCkNQg6eTNLW9H0hfCT4HLJe2atrWHpEI7yG3AmZLGShpIVm+fNx34qKSByhq980H+TmBfSZ+W1D8N75F0QAVFuxMYJul8ZQ3wg9LVXMENZNVEH6HCYJH2dxZwN/AfFZ4Xu5KdV/0lfZysjeiuEmVud38lDZB0uqSdImtbeYOsmhCy43+upMOU2V7S36V2m3IWAUMl7ZRLG5TW/6ak/YF/ameZ0R2sbyrZ1cJ/pPIfBZxE1j7X5zhY1LCIeDEipnUw+wtkv2RnAw+RNSRem+b9lOxL4CngCeBXRcueQXZpP5OscfOXZO0AW+I0svaDV4A7yOqm/5jm/TyVZQ7ZF9Kt7Sxfyv+S/YK9R9JKshsADiu9yGb7CllD5aOS3iBrWN4PICJ+D3yfrE2omU3vvLqcrE1gEXA92d1GpGVXkrUdTSA7Rq8C3wa2oYy07DFkX1SvAi8AH8jNf5jsy/aJiOhM1RjA/wDnpOBY7ryYStaw/zpZQ+6pEbGkRJlL7e+ngTnpGJ8LnJ6Wm0Z2g8aPUhmayQJhWRHxPFmbwux099LuZG1qnwJWkv1fFJ97FwPXp/yfKFrfWrJjfnza558AZ6Tt9DnauArSzDpDUgBjIqK5h8vxJ+CmiLi6Sus/k6zx/n3VWL/VPrdZmG3lJL0HOBjo8qo5swJXQ5ltxSRdT1ZVdn7R3WhmXcrVUGZmVpavLMzMrKxe2Waxyy67xMiRI3u6GGZmW5WmpqbXI6KhvXm9MliMHDmSadM6uuPUzMzaI6nDW69dDWVmZmVVPVgo6y77SUl3pulRqSvfZmXdCxe6C94mTTen+SNz67gwpc/KPU1rZmbdpDuuLL5I1mtowbeByyNiH7InNAtdIpwFLEvpl6d8pC6NJwAHkvXA+hNt4bsXzMysc6oaLCQNJ3tHwdVpWsAHyboRgKxLhFPS+MlpmjT/6JT/ZLL++ddExEtkj/8fWs1ym5nZxqp9ZfF94D/Y0EnYUGB5rlfL+Wzo7ncPUhfHaf6KlL8tvZ1l2kg6R9I0SdMWL17cxbthZta3VS1YSDoReC0iKu07fotExKSIaIyIxoaGdu/8MjOzzVTNW2ePAD4i6QRgW2BHst5Dd5bUL109DGfDOw8WkPVnP19SP7JXhi7JpRfkl+lyTXOX8ejsJYwfPZRD9hpcrc2YmW1VqnZlEREXRsTwiBhJ1kD9p4g4HbiPDS8fmUj2pi/IuqCemMZPTfkjpU9Id0uNIusiudSbwDZb09xlnH71o1x2zyxOv/pRmuYuK7+QmVkf0BPPWXwF+JKkZrI2iWtS+jVkLy5pJnuRzwUAETGD7KUzM8leWH9eevlPl3t09hLWtrTSGrCupZVHZ7fbVb+ZWZ/TLU9wR8T9ZO/5JSJm087dTBHxNvDxDpb/FtnLVqpq/OihDOhXx7qWVvr3q2P86KHV3qSZ2VahV3b3sbkO2WswN5493m0WZmZFHCyKHLLXYAcJM7Mi7hvKzMzKcrAwM7OyHCzMzKwsBwszMyvLwcLMzMpysDAzs7IcLMzMrCwHCzMzK8vBwszMynKwMDOzshwszMysLAcLMzMry8HCzMzKcrAwM7OyqhYsJG0r6TFJT0maIenrKf06SS9Jmp6GcSldkn4gqVnS05IOzq1roqQX0jCxg02amVmVVPN9FmuAD0bEm5L6Aw9J+n2a9+8R8cui/MeTvV97DHAYcAVwmKQhwEVAIxBAk6TJEeEXZJuZdZOqXVlE5s002T8NUWKRk4Eb0nKPAjtLGgZ8GJgSEUtTgJgCHFetcpuZ2aaq2mYhqV7SdOA1si/8qWnWt1JV0+WStklpewDzcovPT2kdpRdv6xxJ0yRNW7x4cVfviplZn1bVYBER6yNiHDAcOFTSO4ELgf2B9wBDgK900bYmRURjRDQ2NDR0xSrNzCzplruhImI5cB9wXEQsTFVNa4CfAYembAuAPXOLDU9pHaWbmVk3qebdUA2Sdk7j2wHHAM+ndggkCTgFeDYtMhk4I90VNR5YERELgbuBYyUNljQYODalmZlZN6nm3VDDgOsl1ZMFpdsi4k5Jf5LUAAiYDpyb8t8FnAA0A6uBzwBExFJJ3wAeT/kuiYilVSy3mZkVUUSpG5S2To2NjTFt2rSeLoaZ2VZFUlNENLY3z09wm5lZWQ4WZmZWloOFmZmV5WBhZmZlOViYmVlZDhZmZlaWg4WZmZXlYGFmZmU5WJiZWVkOFmZmVpaDhZmZlVU2WEjaV9K9kp5N0++S9NXqF83MzGpFJVcWPyV7YdE6gIh4GphQzUKZmVltqSRYDIyIx4rSWqpRGDMzq02VBIvXJe0NBICkU4GFVS2VmZnVlEpefnQeMAnYX9IC4CXgH6paKjMzqyllrywiYnZEfAhoAPaPiPdFxJxyy0naVtJjkp6SNEPS11P6KElTJTVLulXSgJS+TZpuTvNH5tZ1YUqfJenDm7uzZma2ecpeWUj6WtE0ABFxSZlF1wAfjIg3JfUHHpL0e+BLwOURcYukK4GzgCvS32URsY+kCcC3gU9KGkvWoH4gsDvwR0n7RsT6zuyomZltvkraLFblhvXA8cDIcgtF5s002T8NAXwQ+GVKvx44JY2fnKZJ849WFplOBm6JiDUR8RLZO7oPraDcZmbWRcpeWUTEZflpSd8F7q5k5ZLqgSZgH+DHwIvA8ogo3E01H9gjje8BzEvbbJG0Ahia0h/NrTa/TH5b5wDnAIwYMaKS4pmZWYU25wnugcDwSjJGxPqIGJfyHwrsvxnbq0hETIqIxohobGhoqNZmzMz6pEraLJ4h3TYL1JM1dJdrr9hIRCyXdB9wOLCzpH7p6mI4sCBlWwDsCcyX1A/YCViSSy/IL2NmZt2gkiuLE4GT0nAssHtE/KjcQpIaJO2cxrcDjgGeA+4DTk3ZJgK/SeOT0zRp/p8iIlL6hHS31ChgDFD8kKCZmVVRh1cWkoak0ZVFs3aUREQsLbPuYcD1qd2iDrgtIu6UNBO4RdI3gSeBa1L+a4CfS2oGlpK6FImIGZJuA2aSPTl+nu+EMjPrXsp+vLczQ3qJrPpJ7cyOiBhdzYJticbGxpg2bVpPF8PMbKsiqSkiGtub1+GVRUSMql6RzMxsa1JJdx9IGkzWVrBtIS0iHqxWoczMrLZUcjfU2cAXye5Cmg6MBx4he7jOzMz6gEruhvoi8B5gbkR8ADgIWF7NQpmZWW2pJFi8HRFvQ9bZX0Q8D+xX3WKZmVktqaTNYn56XuLXwBRJy4C51SyUmZnVlkr6hvr7NHpxegp7J+APVS2VmZnVlEoauH9A1uvrXyLigW4ok5mZ1ZhK2iyagK9KelHSdyW1+8CGmZn1XpW8Ke/6iDiB7I6oWcC3Jb1Q9ZKZmVnN6EwX5fuQdTG+F/B8dYpjZma1qGywkPSddCVxCfAM0BgRJ1W9ZGZmVjMquXX2ReDwiHi92oUxM7PaVMmts1d1R0HMzKx2bc5rVc3MrI9xsDAzs7IqaeC+TNKBnV2xpD0l3SdppqQZkr6Y0i+WtEDS9DSckFvmQknNkmZJ+nAu/biU1izpgs6WxczMtkwlDdzPAZMk9QN+BtwcESsqWK4F+LeIeELSIKBJ0pQ07/KI+G4+s6SxZK9SPRDYHfijpH3T7B+TvcN7PvC4pMkRMbOCMpiZWReo5KG8qyPiCOAMYCTwtKSbJH2gzHILI+KJNL6SLOjsUWKRk8m6FVkTES8BzcChaWiOiNkRsRa4JeU1M7NuUlGbhaR6sgfy9gdeB54CviTplgqXH0n2HoypKenzkp6WdG16Cx9kgWRebrH5Ka2jdDMz6yaVtFlcTvbE9gnA/42IQyLi2+nBvIMqWH4H4Hbg/Ih4A7gC2BsYBywELtv84m+0nXMkTZM0bfHixV2xSjMzSyq5sngaGBcRn4uIx4rmHVpqQUn9yQLFjRHxK4CIWBQR6yOiFfhpbh0LgD1ziw9PaR2lbyQiJkVEY0Q0NjQ0VLBbZmZWqUqCxXJyDeGSdpZ0CkCphm5JAq4BnouI7+XSh+Wy/T3wbBqfDEyQtI2kUcAY4DHgcWCMpFGSBpA1gk+uoNxmZtZFKrkb6qKIuKMwERHLJV1E9ua8Uo4APg08I2l6SvtP4DRJ44AA5gCfS+udIek2YCbZnVTnRcR6AEmfB+4G6oFrI2JGJTtnZmZdo5Jg0d7VRyXdhDwEqJ1Zd5VY5lvAt9pJv6vUcmZmVl2VVENNk/Q9SXun4XtkL0QyM7M+opJg8QVgLXBrGtYA51WzUGZmVlsqqU5aBbiLDTOzPqxssEhdbnyZ7OnttvwR8cHqFcvMzGpJJQ3cvwCuBK4G1le3OGZmVosqCRYtEXFF1UtiZmY1q5IG7t9K+mdJwyQNKQxVL5mZmdWMSq4sJqa//55LC2B01xfHzMxqUSV3Q43qjoKYmVntqqTX2YGSvippUpoeI+nE6hfNzMxqRSVtFj8jeyjvvWl6AfDNqpXIzMxqTiXBYu+I+A6wDiAiVtN+n09mZtZLVRIs1krajqxRG0l7k3X5YWZmfURFXZQDfwD2lHQjWdfjZ1azUGZmVlsquRtqiqQngPFk1U9fjIjXq14yMzOrGZX0DfX+NLoy/R0riYh4sHrFMjOzWlJJNVT+Ybxtyd6Z3QS4I0Ezsz6ibAN3RJyUG44B3gksK7ecpD0l3SdppqQZkr6Y0odImiLphfR3cEqXpB9Iapb0tKSDc+uamPK/IGliR9s0M7PqqORuqGLzgQMqyNcC/FtEjCVr7zhP0liyd2PcGxFjgHvZ8K6M44ExaTgHuAKy4ELWyH4Y2VXNRYUAY2Zm3aOSNosfkm6bJQsu44Anyi0XEQuBhWl8paTngD2Ak4GjUrbrgfuBr6T0GyIigEcl7SxpWMo7JSKWpvJMAY4Dbq5kB83MbMtV0mYxLTfeAtwcEQ93ZiOSRgIHAVOB3VIgAXgV2C2N7wHMyy02P6V1lF68jXPIrkgYMWJEZ4pnZmZlVHLr7PVbsgFJOwC3A+dHxBvShoe/IyIkRYcLd0JETAImATQ2NnbJOs3MLFNJNdQzbKiG2mgW2ff9u0os258sUNwYEb9KyYskDYuIhama6bWUvgDYM7f48JS2gA3VVoX0+8uV28zMuk4lDdy/J3uC+/Q03JWGE4GTOlpI2SXENcBzEfG93KzJbHhHxkTgN7n0M9JdUeOBFam66m7gWEmDU8P2sSnNzMy6SSVtFsdExEG56QskPRERF3S4ROYI4NPAM5Kmp7T/BC4FbpN0FjAX+ESadxdwAtAMrAY+AxARSyV9A3g85buk0NhtZmbdo5JgIUlHFBq1Jb2Xyp7PeIiOe6c9up38AZzXwbquBa6toKxmZlYFlQSLs4BrJe2UppcDn61aiczMrOZUcjdUE/DuQrCIiBVVL5WZmdWUSl6rupuka4BbImKFpLGpvcHMzPqISu6Guo7s7qPd0/RfgfOrVB4zM6tBlQSLXSLiNqAVICJagPVVLZWZmdWUSoLFKklD2fBa1fGA2y3MzPqQSu6G+hLZA3N7S3oYaABOrWqpzMysppQMFpLqgSPTsB/ZcxOzImJdN5TNzMxqRMlqqIhYD5wWES0RMSMinnWgMDPreyqphnpY0o+AW4FVhcSIKPtOCzMz6x0qCRbj0t9LcmmB38FtZtZndBgsJH0xIv4X+D+pnyczM+ujSrVZfCb9/UF3FMTMzGpXqWqo5yS9AOwu6elcetmXHpmZWe/SYbCIiNMkvYOsq4+PdF+RzMys1pRs4I6IV4F3d1NZzMysRlXS3cdmkXStpNckPZtLu1jSAknT03BCbt6FkpolzZL04Vz6cSmtWVK5t/OZmVkVVC1YkPVWe1w76ZdHxLg03AUgaSwwATgwLfMTSfXpCfIfA8cDY4HTUl4zM+tGFQcLSQM7s+KIeBCo9F3ZJ5O9L2NNRLxE9h7uQ9PQHBGzI2ItcEvKa2Zm3aiSlx+9V9JM4Pk0/W5JP9mCbX5e0tOpmmpwStsDmJfLMz+ldZTeXjnPkTRN0rTFixdvQfHMzKxYJVcWlwMfBpYARMRTwPs3c3tXAHuTPRW+ELhsM9eziYiYFBGNEdHY0NDQVas1MzMq6+6DiJgnKZ+0WS8/iohFhXFJPwXuTJMLgD1zWYenNEqkm5lZN6nkymKepPcCIam/pC8Dz23OxiQNy03+PVC4U2oyMEHSNpJGAWOAx4DHgTGSRkkaQNYIPnlztm1mZpuvkiuLc4H/JWsrWADcA/xzuYUk3QwcBewiaT5wEXCUpHFkHRHOAT4HEBEzJN0GzARagPNS9+hI+jzZg4H1wLURMaPy3TMzs66giCidQToiIh4ul1ZLGhsbY9q0aT1dDDOzrYqkpohobG9eJdVQP6wwzczMeqlSXZQfDrwXaJD0pdysHcmqhMzMrI8o1WYxANgh5RmUS38DOLWahTIzs9pSqtfZB4AHJF0XEXO7sUxmZlZjKrkb6jpJm7SCR4Rfq2pm1kdUEiy+nBvfFvgY2e2tZmbWR5QNFhHRVJT0sKTHqlQeMzOrQWWDhaQhuck64BBgp6qVyMzMak4l1VBNZE9ci6z66SXgrGoWyszMaksl1VCjuqMgZmZWu0o9lPfRUgtGxK+6vjhmZlaLSl1ZnFRiXgAOFmZmfUSph/I+050FMTOz2lXJa1V3kvS9witLJV0myXdDmZn1IZX0OnstsBL4RBreAH5WzUKZmVltqeTW2b0j4mO56a9Lml6l8piZWQ2q5MriLUnvK0xIOgJ4q9xCkq6V9JqkZ3NpQyRNkfRC+js4pUvSDyQ1S3pa0sG5ZSam/C9Imti53TMzs65QSbD4J+DHkuZImgv8iOxVq+VcBxxXlHYBcG9EjAHuTdMAx5O9d3sMcA5wBbQ9PX4RcBhwKHBRIcCYmVn3qeShvOnAuyXtmKbfqGTFEfGgpJFFySeTvZcb4HrgfuArKf2GyN7x+qiknSUNS3mnRMRSAElTyALQzZWUwczMukYld0N9MQWKlcD3JD0h6djN3N5uEbEwjb8K7JbG9wDm5fLNT2kdpbdXznMKd2wtXrx4M4tnZmbtqaQa6rPpauJYYCjwaeDSLd1wuorY5D0ZW7C+SRHRGBGNDQ0NXbVaMzOjsmCh9PcEsqqiGbm0zlqUqpdIf19L6QuAPXP5hqe0jtLNzKwbVRIsmiTdQxYs7pY0CGjdzO1NBgp3NE0EfpNLPyPdFTUeWJGqq+4GjpU0ODVsH5vSzMysG1XynMVZwDhgdkSsljQUKNsViKSbyRqod5E0n+yupkuB2ySdBcwle8gP4C6yYNQMrC6sPyKWSvoG8HjKd0mhsdvMzLqPsqaDMpmyHmjfR9bG8FBE3FHtgm2JxsbGmDZt2mYt2zR3GY/OXsL40UM5ZC/fpWtmfYekpohobG9eJW/K+wmwDxtuV/2cpA9FxHldWMaa0DR3Gadf/ShrW1oZ0K+OG88e74BhZkZl1VAfBA5Idy8h6XpgZlVL1UMenb2EtS2ttAasa2nl0dlLHCzMzKisgbsZGJGb3hN4oTrF6VnjRw9lQL866gX9+9UxfvTQni6SmVlNKPWmvN+StVEMAp6T9FiaPgx4rHuK170O2WswN5493m0WZmZFSlVDfbfEvC57mK7WHLLXYAcJM7Mipd6U90B76akH2tOAB6tVKDMzqy2VNHAj6SDgU8DHgZeA26tZqJ7kW2fNzDZVqs1iX7IriNOA14FbyZ7L+EA3la3b+dZZM7P2lbob6nmy22ZPjIj3RcQPgfXdU6yekb91ds26Vr7/x7/SNHdZTxfLzKzHlaqG+igwAbhP0h+AW9j8DgS3CoMHDqA1Nd0H8NALrzN19hI+3rgnHz14eLtXGa62MrO+oMMri4j4dURMAPYH7gPOB3aVdMUWvM+ipj37yoqNpgNYuz64aerLnH71o5tcZRSqrS67Z1a7883MeouyD+VFxKqIuCkiTiLrIvxJsrfb9TodXTYFG57ozmvviW8zs96okie420TEsvSSoaOrVaCedODuO200fejIwSWf6PYT32bWV1R062xfsWz12rZxAUfutytfOf6ADtsk/MS3mfUVDhY5gwcOaBuPoumO+IlvM+sLHCxyihu475v1GpfcOcPPXZhZn9epNouuImmOpGckTZc0LaUNkTRF0gvp7+CULkk/kNQs6WlJB1etXEXTz8xf7gZsMzN6KFgkH4iIcbm3Ml0A3BsRY4B70zTA8cCYNJwDXFGtAn304OEM6LfhkCx6Yw2tAXVuwDazPq4ng0Wxk4Hr0/j1wCm59Bsi8yiws6Rh1SjAIXsN5uKTDmTk0IGIrN2iDjhin13aqqCa5i7jx/c1+5kKM+tTeqrNIoB7JAVwVURMAnaLiIVp/qvAbml8D2Bebtn5KW0hXaxp7jIuuXMGa9a1ZoFCMKBfHed/aN+2QOG+o8ysL+qpYPG+iFggaVdgiqTn8zMjIlIgqZikc8iqqRgxYkSZ3O17dPYS3l7X2jY9umEHvv2xd7UFBL921cz6qh6phoqIBenva8AdwKHAokL1Uvr7Wsq+gOxVrgXDU1rxOidFRGNENDY0NGxWuV5YtHKj6ebX3uSqB15sq3LyQ3hm1ld1e7CQtL2kQYVx4FjgWWAyMDFlmwj8Jo1PBs5Id0WNB1bkqqu61PR5yzdJu2fmorZ+nwoP4X3p2P1cBdUN3D5kVjt6ohpqN+AOSYXt3xQRf5D0OHCbpLOAucAnUv67gBOAZmA18JlqFWzEkIHMWbJ6k/S317Vy3o1N/MvR+/Kpw0Y4SHQDtw+Z1ZZuDxYRMRt4dzvpS4BN+pyKiADO64aiseKtdR3Oe/WNNfznHc/w8pJVDNquv7v3qDK3D5nVFj/BnbPrjtsCK0rmuerB2QDU14lLTn4n+71jkPuGqoJC+9C6lla3D5nVAAeLnA/stytTZi4qmadwi1ZLa/B/fv0M9fV1tKx3VUlXcyeNZrXFwSJn2eq1bQ/jVaI1oLWldaP3XfhLreu4k8bK+Y2NVm0OFjmdreqoqwMQEeGqkirwF2BlfDOAdQcHi5yfPzKn4qsKgPWtAEGd4MzDR/oftIzOfPn7C7ByvhnAukMt9Q3V4373TOnHN+oE++y6wybprQFX/Xl2xc8D9IbnBy696zmO+p/7uPSu5yrK3zR3GRMmPcL/3D2LCZMeKbvvfmVt5fywqHUHX1nkrFtf+rqiNeDF195sd14EfPv3z3Hbue8tuY6mucv45KRHaFkf9KsXt55z+Fb3K/DSu57jynRXWOHvBSccUHKZqx54se34rlsffOX2pzfqSqVY/gsv6HwVYV/imwGsO/jKopNKhZNpFVwpXPXAi7SkL82W9cFVD7zYRSXrPjc99nLJ6fZMnbPxlUHza2/yyase4aapG5YtXHHdNPVlvnrHM7Smg90aWRVhb7gi6yo+FtbdfGXRhVrbiSTF9fSPzVm60fxFb7zdTaXbfMX7sHpty0bzC9M3TX2Z3z+7kOPfOWyT509WvtWyyXpbWoP/Sg86jhi6PV/7zbOsb412A/Kvp7/CXc++6tuU2bQ952snHsjFv53R9kzKzf84HqDt+OfH++oxsy3nYNHF9r7wdzQM2obxo4eyYPlbPD4n++VXLxi9y/YsX73xU+JL3lzTNt40dxlXPvAir73xNp98zwg+ddjm9Z67Oc6/5Unu/+tijtq3ge9POGijMhU3NLe0brxsSyuccc1UHnzhdQD+nP4WDNqmvt1ACtmVWqEqq5y1acNrc20Y+S/Bnjx+1ZYP2L96Yn5bN/rrWlq59fGXNzo2X/nlU7y87C1a1rfSr04gOcjaFnOw6GLrI+sa5NfTX9kk/YXFqzbJP3/524z7+t1sv21/Fi5/q+1L9an5zwDwqcNGlLyLqHhe4QvzpcVvMrphBz535N5lvxzOv+XJtvIW/n5/wkHZ+z1+O6Ot2/ZCH1ntebAoQOStXLO+5PY7qzWyHoIv/+NfaVkf1NfBISMG8/icZW1XJU/N77hrlqa5y7j9ifkIOHD3nVi2em2X/+rOfy5Q/pd9R/kBbn9iPrdNm8f6tK9IbftZX1/XFigKmnPn2dr1QaHydM26Vm5/Yn7beeKrDesMZV0v9S6NjY0xbdq0Ti838oLfVaE0m69/vRg6cACvrtxw9XHu+0dzwQkHtH3h/bJpftuvxq+deCBf/fUzG/2Kr0tXNKMbduCo/Xbl2VdW8KeZi3h91RqGDx7IZZ8Yx4RJj2zSuN+ZhxO3BgK2H1CP6sTKtzetEgMY07A9U/7tqLbpwhfq4IEDuG/WayWvWIq/7D92xV/a5tUpC3D1gm+c8jcbBafzb3mSu55ZyNr1gVLe/EdR7nMYtE19p4PxXkMGMnfphg4zjx27G587cm+AtiD60YOHl/1h0luU26/C/JVvrWPGwjc4/p3DetVVa56kptyrrjee52CxQa0FC+sZew0ZyGsr3+btVNVTbMjA/uw5ZCCffM8IXl6yil9PX8CiN9Z0OrgOqFf65V+bDh2ZfXHOfn0Vq9eu5+1164n0TvqjD9gQYAoBtdQVWiWBpqM8+aBdyTYGDxzAs6+soHnRSta0tDJql+156fVVrG1p5eWlq1nT0so7d9+RYw58By8sWsnkp15pC+b77jaIV1a8xao16xmyfX8+etBwrntkzkYvRQOor4P9dhvEN075m7ZjkL+6Lw66lQakzgbirg7gDhYVcrAw23JDBvbnjbfXUSdRL7GuNWhJl7v968Xho4fy+JylbcF45+36EcCK3E0Q575/NI/MXsK8patZunrT3qAPHTmYBcvfYsmqtZt8kdea4h8F2/WvY0D/Ota1BHWw0dWugPekfUPiwGE7tlUlF4JQ86KVLF21lv71dfz1tTdpbQ3q68TZ7xvFG2taOrwyrISDRYUcLMyst7j9n97b6YBRKlj4OQszs17oU5Me6dL1OViYmfVCa7q4PWyrCRaSjpM0S1KzpAt6ujxmZn3JVhEsJNUDPwaOB8YCp0ka27OlMjPrO7aKYAEcCjRHxOyIWAvcApzcw2UyM+sztpZgsQcwLzc9P6W1kXSOpGmSpi1evLhbC2dm1tttLcGirIiYFBGNEdHY0NDQ08UxM+tVtpZgsQDYMzc9PKV1qTmX/l1Xr9LMrEd09ffZ1tKR4OPAGEmjyILEBOBT1diQA4aZ2aa2imARES2SPg/cDdQD10bEjB4ulplZn7FVBAuAiLgLuKuny2Fm1hdtLW0WZmbWgxwszMysLAcLMzMry8HCzMzK6pXvs5C0GJi7BavYBej4pdLm41Oaj09pPj6l9eTx2Ssi2n2quVcGiy0laVpHLwAxH59yfHxK8/EprVaPj6uhzMysLAcLMzMry8GifZN6ugA1zsenNB+f0nx8SqvJ4+M2CzMzK8tXFmZmVpaDhZmZleVgkSPpOEmzJDVLuqCny9PVJO0p6T5JMyXNkPTFlD5E0hRJL6S/g1O6JP0gHY+nJR2cW9fElP8FSRNz6YdIeiYt8wNJKrWNWiOpXtKTku5M06MkTU37c6ukASl9mzTdnOaPzK3jwpQ+S9KHc+ntnl8dbaMWSdpZ0i8lPS/pOUmH+/zZQNK/pv+tZyXdLGnbXnMORYSHrN2mHngRGA0MAJ4CxvZ0ubp4H4cBB6fxQcBfgbHAd4ALUvoFwLfT+AnA7wEB44GpKX0IMDv9HZzGB6d5j6W8Sssen9Lb3UatDcCXgJuAO9P0bcCENH4l8E9p/J+BK9P4BODWND42nTvbAKPSOVVf6vzqaBu1OADXA2en8QHAzj5/2o7NHsBLwHa5z/XM3nIO9fgBrpUBOBy4Ozd9IXBhT5eryvv8G+AYYBYwLKUNA2al8auA03L5Z6X5pwFX5dKvSmnDgOdz6W35OtpGLQ1kb2C8F/ggcGf6wnod6Fd8jpC9W+XwNN4v5VPxeVPI19H5VWobtTYAO6UvQxWl+/yJtmAxjywI9kvn0Id7yznkaqgNCh90wfyU1iulS96DgKnAbhGxMM16FdgtjXd0TEqlz28nnRLbqCXfB/4DaE3TQ4HlEdGSpvP703YM0vwVKX9nj1mpbdSaUcBi4Gepqu5qSdvj8weAiFgAfBd4GVhIdk400UvOIQeLPkjSDsDtwPkR8UZ+XmQ/Tap6P3V3bKOzJJ0IvBYRTT1dlhrWDzgYuCIiDgJWkVUJtemr5w9Aakc5mSyo7g5sDxzXo4XqQg4WGywA9sxND09pvYqk/mSB4saI+FVKXiRpWJo/DHgtpXd0TEqlD28nvdQ2asURwEckzQFuIauK+l9gZ0mFN0rm96ftGKT5OwFL6PwxW1JiG7VmPjA/Iqam6V+SBQ+fP5kPAS9FxOKIWAf8iuy86hXnkIPFBo8DY9JdBQPIGpwm93CZulS6s+Qa4LmI+F5u1mSgcEfKRLK2jEL6GemulvHAilQVcDdwrKTB6dfUsWR1pAuBNySNT9s6o2hd7W2jJkTEhRExPCJGkn32f4qI04H7gFNTtuJjU9ifU1P+SOkT0p0uo4AxZI227Z5faZmOtlFTIuJVYJ6k/VLS0cBMfP4UvAyMlzQwlb9wfHrHOdTTjUK1NJDdvfFXsjsO/quny1OF/Xsf2eX708D0NJxAVud5L/AC8EdgSMov4MfpeDwDNObW9VmgOQ2fyaU3As+mZX7Ehl4C2t1GLQ7AUWy4G2o02T9qM/ALYJuUvm2abk7zR+eW/6+0/7NId/OUOr862kYtDsA4YFo6h35NdjeTz58N5f868Hzah5+T3dHUK84hd/dhZmZluRrKzMzKcrAwM7OyHCzMzKwsBwszMyvLwcLMzMpysLCtnqT1kqannj5/IWlgDZTpKEnv3cJ17C7pl51c5kxJP0rj50o6Y0vKYFbgYGG9wVsRMS4i3gmsBc6tZKHcE6/VcBTQqWBRXJ6IeCUiTu0ofzkRcWVE3LC5y5vlOVhYb/NnYB9JJ6X+/Z+U9EdJuwFIuljSzyU9DPxc0khJf5b0RBrem/IdJekBSb+RNFvSpZJOl/SYsvct7J3yNUi6XdLjaTgiddJ4LvCv6Yrnb9vL11558juSyvZsGj9T0q8k/UHZOx2+k8v3GUl/lfQYWfcS5Nb95TS+TzoOT6X9LJT/31N5npb09ZS2vaTfpbzPSvpkVT4p26pU85eVWbdKv8yPB/4APASMj4iQdDZZb7L/lrKOBd4XEW+lKqtjIuJtSWOAm8meIgZ4N3AAsJTsnQtXR8Shyl4a9QXgfLL+oy6PiIckjSDrtuIASVcCb0bEd1PZbirOl9a9UXnK7OI4sp6C1wCzJP0QaCF7avgQsl5L7wOebGfZG4FLI+IOSdsCdZKOJetK4lCyp60nS3o/0AC8EhF/l8q+U5lyWR/gYGG9wXaSpqfxP5P1f7UfcKuyTucGkL2HoWBy7ou5P/AjSeOA9cC+uXyPR+oWW9KLwD0p/RngA2n8Q8DYrCsgAHZU1qtvsVL5JlcQKADujYgVqTwzgb2AXYD7I2JxSr+1aB+QNAjYIyLuAIiIt1P6sWT9MhWCyw5kwePPwGWSvk3W7cmfKyib9XIOFtYbvBUR4/IJ6Vf39yJisqSjgItzs1flxv8VWER2FVEHvJ2btyY33pqbbmXD/04d2RVMfjlyQYEK8q0qztyBfHnWs+X/vwL+OyKu2mRG9grUE4BvSro3Ii7Zwm3ZVs5tFtZb7cSGbponlsm3MCJagU+TvbqyM+4hq5ICIF2hAKwke3VtuXxbaipwpKShyrqf/3hxhohYCcyXdEra9jap+u1u4LOFKxxJe0jaVdLuwOqI+H/A/5B1Q259nIOF9VYXA7+Q1ET2ysmO/ASYKOkpYH8q/5Vf8C9AY2ognsmGO7F+C/x9oYG7RL4tkqrJLgYeAR4Gnusg66eBf5H0NPAX4B0RcQ/Z+8YfkfQM2fspBgF/AzyWqvYuAr7ZFWW1rZt7nTUzs7J8ZWFmZmU5WJiZWVkOFmZmVpaDhZmZleVgYWZmZTlYmJlZWQ4WZmZW1v8HANQQ3VaCT28AAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(torch.arange(0,len(flat_fft)), flat_fft.abs(), '.')\n",
+ "plt.title('Model Fourier Frequency Representation') \n",
+ "plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"Parameter_Frequency.png\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Normalizing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(-0.0023)"
+ ]
+ },
+ "execution_count": 56,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.mean(conc)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "conc2 = conc #+ 0.0016"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(-0.0023)"
+ ]
+ },
+ "execution_count": 58,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.mean(conc2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 59,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat_fft2 = fft.rfft(conc)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 60,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[<matplotlib.lines.Line2D at 0x7ff775a99c10>]"
+ ]
+ },
+ "execution_count": 60,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEFCAYAAAABjYvXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABYNklEQVR4nO29e3QU153v+/1VtwQIhBDijZBAGGMQjjHCRtiOH7Gd2Ll2cEwc/JiZ+MTPM8m5kztnZo1PMuGySO4c587kjues43tt7Hg5k2NsYmNj7LGTGL8fgEEEjIQtHgIJ8UY0QiBQq7v2/aNql6p2V3VXv1ut32ctFuru6q5du/bev/17FgkhwDAMwwxttHw3gGEYhsk/LAwYhmEYFgYMwzAMCwOGYRgGLAwYhmEYAMF8N8CLcePGienTp+e7GQzDMIOKpqamk0KI8cl+r2CFwfTp07F169Z8N4NhGGZQQUTtqXyPzUQMwzAMCwOGYRiGhQHDMAwDFgYMwzAMWBgwDMMwYGHAMAzDoIiFQVN7CE++vxdN7aF8N4VhGKbgKdg8g3Roag/hvmc3IRzRURrU8MKDjWiorcx3sxiGYQqWotQMNrV1IRzRoQugP6JjU1tXvpvEMAxT0BSlMGisq0JpUEOAgJKghsa6qnw3iWEYpqApSjNRQ20lXniwEZvautBYV8UmIoZhmAQUpTAADIHAQoBhGMYfRWkmYhiGYZKDhQHDMAzDwoBhGIYpcmHAiWcMwzD+KFoHMieeMQzD+KdoNQNOPGMYhvFP0QoDTjxjGIbxT9GaieIlnjW1hzghjWEYxkZGhAER3QLg3wAEADwrhHjc5ZjvA1gBQADYIYS4NxPnjodb4hn7EhiGYWJJ20xERAEATwK4FcBcAPcQ0VzlmFkA/huAq4UQ9QB+ku55U4V9CQzDMLFkQjO4EsBeIUQbABDRSwCWANhlO+YhAE8KIUIAIIQ4noHzxqWpPYS12zpBAO5cUG3t/qUvoT+isy+BYRjGJBPCYCqAg7bXnQAWKcdcDABE9CkMU9IKIcQf1B8ioocBPAwANTU1KTeoqT2Ee1ZtRDgqAAAvN3XixYcaLbMRF7FjGIZxkisHchDALADXA6gG8BERXSqEOG0/SAixCsAqAFi4cKFI9WSb2rrQHx34ujQHyYWfi9gxDMM4yURo6SEA02yvq8337HQCWC+E6BdC7AewG4ZwyAqNdVUIBMh6zeYghmGY+GRCGGwBMIuIZhBRKYC7AaxXjlkHQysAEY2DYTZqy8C5PZEXRgB+eNV0NNRWcnkKhmEYD9I2EwkhIkT0YwB/hOEPeE4I0UJEKwFsFUKsNz/7JhHtAhAF8PdCiKyF8djNRALAqo8NufP8xgPo69cR0Agrl8zDvYtS90swDMMUExnxGQgh3gLwlvLectvfAsDfmv+yTmVZKYgAYboNdGEIBCEM4RDRBf5x3U4AYIHAMAyDIixH0dQewso3W6Ar7mchABpwI0AXwPLXm9lkxDAMgyIUBjKpTKUkqOHhr9dBcwgEwUlnDMMwKEJhIJPKbGs+CMD3Gqrx2Lfn4Jd3XIoAGe8FNeIoowzBznmGGdwUnTCQSWU3zZ1ovScAzJtSgab2EJoPd0OT6oHdbsSkjKz39Os/teK+ZzexQGCYQUjRCQPJ/pPnHK8/aD2O+57dhBc3d6A/KiAARKNcmygTcL0nhhn8FF0J66b2EJat2ohI1OlBPnbmAsIRHfJdAiejZQqu98Qwg5+iEwavbuuMEQQEYHFdFVqP9aA/YuQZ3LVwmqOAHZM6XO+JYQY/RScMvAoaPb/xAO5fPB0tR87g1nmTOb8gw3C9J4YZ3BSdMJg3pSLmPQHgQr9uZSJvbutCy+Fu1gwYhmFMis6BHOoNe36mC+NfOCrwwuYO3PMMR74wDMMARSgMGuuqEPAZMRqO6Hh1W2d2G8QwDDMIKDph0FBbiYe+Xuf5uSonUn5oAsMwTBFRdMKgqT2EZz52r45NBDxybR1KAwQCUBogLF1QndsGMgzDFCBF50Beu60TUY/tPgngTF8E31s4LebZyAzDMEOZohMG8dwFOoCXtx5EVBcoDWq4s8i0gqb2EMf6DxH4XuePYu37ohMGdy6oxktbOhCNLVwKANZDb/r6dazd1pnWzSykQSHrA4UjOkqDGl54sDHvbRpMFNK9TATf6/xRzH1fdD4DYOChNnGPgaElpBpaWmjF2bg+UOo0tYdwzzOb8C9/bB0U4cZ8r/NHMfd90QmDTW1dMQ+28SISTf15BoU2KGR9oADlr+bSYC1j/eq2Tqtu1WAINy6Eez1UKea+LzozUWVZaVLHp3ozC604W77rAw1m9VndO2Qy3Dgb5if1XgPAk+/vTekcg8k8Vgjke55lk4wIAyK6BcC/AQgAeFYI8bjHcUsBvALgCiHE1kycW6X5cLfvY9OZ9IU4KPJZH8hNUyqEPvHD0gXVeGXrQfRHBUoyGG6cTQEp73U65xjMAjyfFGsdrrSFAREFADwJ4GYAnQC2ENF6IcQu5bhyAH8DYHO654zHyZ6+pI5Px4lcaIMin7u8QtOUkqGhthIvPrw4432XCwGZzjkGswBnMk8mNIMrAewVQrQBABG9BGAJgF3Kcb8A8CsAf5+Bc3oyrnxYUsfvPdaTpZbklnzv8gpRU0qGbAj2XAjIdM4xmAU4k3kyIQymAjhoe90JYJH9ACJaAGCaEOI/iMhTGBDRwwAeBoCamtRKTC9dUI01n3d4Jp6p9EU8YlALFK/dfyHs8gpNU0qGXNj2s9E36Zwj1e+yn8E/g6mvsu5AJiINwP8D4P5ExwohVgFYBQALFy5MyaTfUFuJGeNGYu+Jc4kPBlAxoiSV0wDI/Y2Ot/vnXV7q5MK2n03SOUey321qD+GeVRst/8qLDy8u+EXOjVzM3Xxr68mSCWFwCMA02+tq8z1JOYB5AD4g4wH0kwCsJ6LvZMuJPHZkKeBTGHSc6k3pHPm40fF2//k20wymHZBKIWhVuSSde7V2WyfCptodjoq0EzeTQba7sqwUod5wymMtV3N3sI2rTAiDLQBmEdEMGELgbgD3yg+FEN0AxsnXRPQBgL/LliAAgJ4LEd/H3lI/KaVzbGrrQl+/GZven5sbnWj3ny8zjTq5lt9Wn9ZkzTWDQatKZQFXv9PUHsLabZ14pakTkWhqC6Fa7qXlUDea2kNZv89yjMk5RwACGmHlknlJP7UwV4v0YBhXdtIWBkKICBH9GMAfYYSWPieEaCGilQC2CiHWp3uOZHj8rS/x5VF/TuERQQ2PfXtOSuepLCu1QlN1+M9vSGdXlmj3n6/duX1yhft1LH+9GboQBaEa++mTQteq3HaygLFL9yq46CagV77ZYi2mgPtCmKgtdy6oxstNRpIeAOw81I37nt2U9fssx5hsuwAQ0QWWv96M2ZPKkzp3rhbpfI+rZMmIz0AI8RaAt5T3lnsce30mzunFbz5xL1/txtgkI4/sqPkMfvIbMqGeeu3+82mftE8uIkJUF1Y2bz5V42T6JBtalR9B5KeNqha6dlsnXtl60DLXvNzUiRcfcn5P3f2+3XzEsZgSYjNo/bSlobYSLz7UiJVvtGBHZ3fOTCByjIX7ddhDPnQxUEXA76KbiUXa78ZrMAVVFF0Gcn8SwUFHTp9PWcVV1WU/D1fLpnqaT/uknFxrt3Vi77EefH7AKEehi+QzwlMhUxFWmdSs/AoiP21UtdCTPX1WwUXA/Xvq7vfWeZOx5cAp9Ed0BDTCXQunxWgUyfTXl0fOWH8HAtk3gdgX8J7z/Xj2k/2W9llZVmqZkPyajtJZpAebY9gvRScMkkEXqSedSXVZTjY/5bCzqZ7afzugEQ6nIehS5dVtneizSWMN8Z9JnQlkkTnZp/YdcmNdFYIaoT8qENAobn9neoL7XVj9jIlQbxgaGeNVIyOXpiRAlmZQYi6I9pIUbrvf2ZPK4wo7v+NzU1sXImYBMALwvYbcPBfEvoDfXD/Juha75hTRBX6+bmfSpqNkyNbGK99BGEUnDOSk8UuqSWdSXU7m5mXKhug2aOy785e3HsTqzR2u5oNsodp0ASAYMBbgTA9y++/JInPAQJE5+zmiwrAvJ8o7yURAgL1dfhdWP2OisqwUGhEAYye8dEE1li6otnwG9VMqsGJ9M8JRgQABv7jjUty7qCZm95toN9xQW4nlt9Xj7eYjuHXeZM9j1WvLxdMCpfNb+kjU9mg0cI+jLpu8TI7BRPc2VWd/vrWNohMGyQgCADjV2+94vXpzhzUZklE1cyXV4w2ahtpKvLqt0zIhuC2O6Zw30YIV0/dEaD3ag5VvtmRskKvXf+2s8Y7P7U14+sN9iJqNiuoCT3+4D6v+aqHr76YaEODVrhcebPQt+OMt0k3tIax8swVRXUAjWNdr/85PX9tpaQlRAfw8Baeq/VzhiI4tB055/kaqm5pU54jMbZDXuGbrQWhEjoioG+dMxJ92HbO+Q8r3M73Q3rmg2iGYpFYGIOlzNbWH8MSG3XkPQy06YaABSCaneMa4kdbfqzd34Kev7QQAfLznJAD4CluLZ6pQj0t3UCZSUb0qcKYjrPy0u8XFgR6NDjguMzXI1esfVz4MpQFyLTJ37MwFx3fV1/bre7v5CIy9t6FdJmvecrsvP7rhorQntF3jigrgnV3H8NGeE47wXbUel66LlPo5GfNHsjb3dMb+prYuh48kEhUgGEEKsp2PXDcTH+w+4Wq2zaRZR72O+ikVjs3O0gXVSfup7CGzGuWvNHbRCQNrRvvkhtkTrL/fbj7i+Ozt5iMxwsBtUU1kqpD4GZSJFu1EKurSBdX4/ZYORHQgqBmv0xVCidrd1B7Cy1sPOr5DMByLdsdlJga5m4li6YJq1z5bdkUNdnTudLxWcZuMpQna6Ra/f+j0eQQ0goiKlB2qbvdeXq9sn4DxlD4ZvhvUKGbzQ5Sa4z6bPq10FuTGuiqHjyQYIGhEiEYH2hnPbJvMdXnNP/n+4dPnccH0i/X1x252BJBUH9qFvQbg6ovG4Sc3Xcw+g0yQrJno/1w/oFLfOm+ypREAwK3zJjuOtaurGoCF0ysxa2I5Tig7M68m+LE1uiVvqRmXiVR0TdNAug5NM55dlO7OKFG77Q5FZ0cIzJ5UntFYa6/rd/vdexfVoKPrHP7QchS31E+yBLt9wqu+jkunVmD57fVxzTZ2LXDF7c74fTKvOx5upkgvgS2v96kP9+Ed0wwiACt8tz8qYsabLoAVb7R4mnm8FrxEYysd7TIdQdNQa1SVVX0G8rWf7/sZg173wP6+/XwCQNXI0pjNybwpFdb9BeI/a0Ltl3wJAqAIhUGy9EcHVGo5Mb18BvZUfB3A5wdC+PxACMEAIRggRBVThTp57IOysqzUio+WN98teUtOerljtS8Sduw7l0jUjKyI6tb5gwEz0iiFXavdsVg1shRPbNiN+smjUT6ixOEstceAy8iOVM0lqsMwGUeo/Tee33gA4YiO5zcewM1mtvmypz+zNKeVSy6FphF08762HDljPenM7RyqFrhmS0dMMlQ0jplGNUV2dJ3DY9+eY0Vi2U0fdkE3f9oYvPvlMehiIPtWjg034mmebiZN+1j90Q0XufZlOjWJ0g2ekPfcXpJC3ou12zqtpDov7VcdM/brBWDNHbdNk31eqnSdCzuuq/VoD36+bid0AWzcdxKapsXN9s5UUEkmGPLCAPD/tDOvXUg0KnDPohpMHTPCuqGrN3e4ZuLKm+22A1GTt3QxMNkTmZbk79mjqRxx/nK36ucB0QrSsXjBFjb68Z6TIADDSgacpU9s2I1P9py02qxR/HBO++/bJ8PqzR34+bqdVnRIqlFRbhFC2w+ehrmWI6IDr/25E8I2yyNRgRc2d2DNloOu8erHFS2wNKg5BKGbzdd+faopctXHbaipGomXtx60+s1NYKs7yGljRmBPnPpbbuGmgLtJE0js9FRrEv3DKztQN34UxpcPc82AdiNZP4MdtZSGZs4PNanOj/Zrny9BjQDTGR3UCMGA5jA/Ac6EN2mqk9RPHu0QVPZxG9EB6EZf29vktknMpxCQsDAArAnRerQnrgP5zgXVWLP1ICJKnKJUD+2RRfZBoWbieplt5C5h7bZOnOzpwwe7TyAS8V5kJPbfs6/1Ms5fmnES7VrtuJlSVOy72B/dcBF+ctPF2HLgFML9OjQz+Ucu7l7alirIpleNRNvJc45dWKpOP7cIoeOKE/ngqV7XHV9EF/jHdcZYuHdRjdUf6obg4onleOzWOdZuVa3JpJoe7l883WGKFMLQRBPF7dvNRX/uCHkKgitN06Xq2JQLvCrMjvf0+TIjqte998Q5qzLwmq0HsSaL1UtlH9o3I7oQIBqYF9I3Fe43NlLxfCaO640KyC1XJCpw96Jpjk0dYPT9/Yun4+mP2hyCgACU26oeb2rriglhDmgECGEJ54f+fSve++o4hEe5lnzmGrAwAPDC5g6s3daJ2RPLHe+rDuSG2kqsMW2XMoLDbWe0dlunY1CQskOOZz9tPdqDlzZ3QAcQIOCeRTWon1LhWGTUAaMmnOkwtBUZ5996tMeKUycC/tRyFJVlpa42dNVGKhcwtwWT4BRQdmFGAGZPKk8YoWWfmLqAa+lxv04/dTEO9YYd8QTNh7uxuK4KOzoHIp8a66rwh5aj5iJiHGvXrJa/3gwA1sIa1AglAUIkKqyoldajPdjU1oVb5012mFjcQgbP9EUQoIGYeCLEONm94vZbj/ZYfgM3CMBFE8vxf333Ujz5/l7rvBf6dTz14T4881cLMUEpwTKhfJgvM+KdC6rx4ucd7oIzy9VLpYanIgSgaYTlt9Vb/qFVH7dZNYsA92jAyrJSEBFICGgaEDV/Wgcwb0qF64ZllSIIAGOhrywrdYy/oAbY900PXTMD5SNKUFlWauWCSNRNYr5zDVgYmPRHdEwYPRzAwEKhOpABp33fS3qru6gbL5kQY790S+5pag/hH1/badndo8LYud1pZo7KY9wGjF2jeL/1uNmQgTj/qC5AACIC2NHZbUXZzJ5UHvN76k6xxVZ6wE5tVRkevnZmTB/YbbmqgF2zpcOxYMuFyE3zAIDqMcPx1zfMAuDuiPOKBnrhwUaHZgAAL33egZvmTLQEBAGYNbEcV86osvwhn+w9iZNnB8JKdSEcJohIVODS6grMm1phCQK7sHvu0/344dUzrH5VQwYJzgQ4XQCf7+9yxK17LQCqiUlFAHilqRNLF1RbiVhy8X5n1zGs3tyB+ikV1vslARoIwUxgRnyn5Wjc4IxEjtx0nc+a6SNRieoCod4wVm/uwKqP26w2qkXs7Av2ivXN1m8Jgbghxas3d+B/vLvbNWJLF8L4LSEQNX1Q37hkIN+BAJzpi6B8RAlaDnc7wmMle4714C9/sxm3zpuMUG84r7kGLAxMAhrh0etm4obZE+ImnUlfQFQXVh2U2ZPKHc7O+ikVju9cbwtfVW2fm9u60HK4G3ea4ZHqoNt/4qzDcXfXwmmembKv2KpJAsaAsjs41aG4ZksHvlk/KWYAqppL/eTRDtOGpONUL1a+6YxaUQWJKmB3HTmDnYe6rWip5sPdiOremSGHTl/A8td3QoBcVWs1Gsh+HYdPn3f8li6Ad788hpLggF24sqwUK95o8RRGRMCIkoAlsHQYwvTLI2dw54LqmAV67/Gz+OlrO/HNuRMd7ZpQPgz/+40XG9FVmzsc33l9+2EAxhisn1LhuQCo0W5uRG1BA6OHlziSKp/7pA0dNrOYbJvdjBiJCjyxYXdMVMsfWo56nrPU1JDcnLKpJmKpiDiSqOd8P/71nd0xwkqaRO3n14gckW+6cJpy7FqRXau1Y0SMGZqEfacf0Q2zo9VmAGu2HIQwQ4ADpkZpP/c6895/vOckrp01ztLg85FrwMLARIexoz3XF8HBU73o6Io1VzS1h7D89WZrMEV0gZ++ttOxA3u5qRPXX+zMipUVTZvaQ1YkiyRsOixf3noQP7x6Rsw5x44stUwn4ajA7mM9rpmydsegRJjnlk4xIZwJeTKePmBG02hm/R7V3NPTF3GYW6ZXlVmLirqD6TlvLD7ShGQXsCNKAthgRsSE+3X8/PVm192eeg3GZZnOS1PjUMs+uDlwn/pwn+vvfa+h2rILu/WbHV0HNnx5LGbnKx/s4rVAHztzAUFtIDb+6Jk+rHijBStur49JjJQ9kKgkszSFvLT1IE4rmfOSQEBDz/l+LHt6Y0y4b/f5fufiFRX4r7/fjlvqJ8Fcg6AD+HTvSWw5cMqxaN9SPwlPfeSsCEwY0A4BOJyyUrCUBDXc1TCQiNVnmqzmTxvjW0tYu60zbiLpxrYu13EkACtqb8AUGXucrgt8rboCy66ocbRnzZaOmGMDGuHqmVX4yEMoH1N8MvYM+GVXGpvLz/efwt7jZ2O++5EZlBEwTV/sM8gTMopEIge+/XkHXvH09rfCET0m01X6F3719pfwWnfCUYENXx13vEcAxiiOsNA5pxorBY3qGLTapgPLrjAWv57z/Y4Jvf3gabzTctRSX/ujAq1He2IiTzRyahUTyofh0OnzjgSrpvaQIxYeAK6cPtZyjEsn7Ed7jCxRAAkFgewD1Wn3sunEl6GRy2+rx5otHRgW1DBrYrllalHt4wCsLFF5jW4Cw46AdwAWYcAm/dwnbQ5/x7IravBB63FHiQQZ9WL/uXGjShHq7bf6IqILrHyjxTXXYfXmDjz7yX73nA6T+dUVeObjNtdaTF3nYrOqD3T1xizybkL+sW/PwdEzF6ydLGD0TXuXoR06Mm9tuQ/hiI49x3oc2sg7u45hw65jvkNU1QxrlYmjh0PTzriOJ2mSdCt/bb+OHZ3daD3m1HInKlrt1MoRuP7i8Wg+1O3yKwYXjR+Jpt6w02cIOHxBL30eK2TsbRFCZL3Aoxtazs84iFi3/ZDjtd+szsV1VQjYevaD3SewenMHtpilnb04H3Y+oU0AlhMUMGy8deNHOY6RnzV3nnb9zWCAMM80W8kdviQSFTHqvzR72HdTqgD7/EBowP4pDAFy37ObYpybH+05idWbO9DUHsKT7+8FANy/eDqmjS1DdWVZTFsvGj8S40c5+3hK5QgEA0arA2T4W+RiE47oePrDfVixvhk7Orvx+YEQVm/uwDstR9HUHooRkN+cOzHGRJHI1g0Y2kbA5cDyYQN7qZHDgtDMY4IBwuxJ5TjtMqHrJ492CIOus2GQYsDb0dmNe57ZhKZ2Y7w0tYdw11Of4aev7YwrCABDwKuCgGC0P5moYreInHPhaMxxAsZu/3hPH0qDGgJmhI+dUy5CSGBAu5LjQ16vnab2EN79Mr7TvKw04CoIAoqme/WscbE/YENqnbItj1w3c2DsacCJnj68+HlHjDCQlxsMEO64vDqm/6+YXmmNu1e3dSZMjNXFgIadS1gzADB1zHAcOh1bt2baWOeC5Vdaf7j7hGM7G4nG7ghVAppRfVJth12A6EKgzlZLCTC+s3pzB46ccd89XT97gpUha4ZUW4tCSVCLUf+l09zuN9CFe1a1gLG7lw5WN577pA1tJ84ZJhx4140KaIR5UyscO08AOBQyyjwAMBqvcOzMBYfpQ8DQ6n7z6X6HfVYDcNm0MSmp3pdONRzGqq1/1cdGv6k7a6Ebi9znLsK/7aTT/DhgBnMiI00A4PtPf2ZFvCQirKxEV06vxHWzJ8RohYmI6CImi3n/SfdwVgFjzK+4fSBjfvn6ZkTMiLa68aNco8QA4M/tIat8ikbAL82KqxK3cE313O8pGrVk3pTRjvDQRD4XXRhaZ1Qf8E2t/M48y8T5zq5jrvPgkknlON8fxfxpY/BP/7Er5vNT58JWO/b4qJIsx3BN1cikH+mZDiwMAIwbNcxVGHz3cmeIn19prT52UyPydMJKojpwoT9256Ues0HZJX3Qehzn43zvUKjXcjjbJ1WAgBW3GyF5NVUjY5zmrUd7MHtiOSaOHo66cSM9FxIZ4725rStmIQKcoaLx1rO6qrIYQSCx213VBdZrZ69GbmiBgWc8ALAiS979ynvXKZk3tSImKAAwFo/VLip/SVBzNW0IeBfLc0Nm2foVBCoaGdFSMldENbklQq2zNbasxPPY/oiOUG8YP7rhIjS1h6DBuDcajA2J9BWp2OeKLoB/VJ5F0FhXFROuqSI8VJ4Z40Ziyf/8xBrDqqavQjC0ZWFez9ptnZapNF7fyWs40NXr+vm+k+fQ1B5C69Ee1w2CF2u2dLAwyDX2uHM7aiXOjeZOLVkunjgKPX2RhMd5OaUkAY1iBEbbyXO46ZIJnoJm15Ee1wVTF4a/oak9hFBv2BE94oyi6Mb86tiFULZHxng//eE+tJ9ynwx+UHfMfmn12GkFNILQhSWABjKLOwAySocQIaHKDhjmgZ+5RJUAsQJ8ZGkA//7AIvzq7S9dj58xbqTneFNpOdzt6QvyQ0AjrDazqa+aWZWUIJDYv6P6r9Tjes7348n39xplHWx+qFBvGDcpJaa90AViSnHc9rUpnhsFALhxzkTXzwfe89ffGsEYG7oRTEGAI9EtVYR5TZuSXD+MSLzckRFhQES3APg3AAEAzwohHlc+/1sADwKIADgB4IdCiPZMnDubqJNHDZP0y64jPZ6LViImlQ/DUXNBIAiUDy8BMLC7HFtW4siCdMPLxPP7rQetEFd7yJ8aLrndY/HSdWEJlHQEAZD44TOe3/NYzRfUjEHLoW70KpPZHpnk14YebxEjcu4Z5V8HXKLRAMR1Pqqc6Onz5dPwQmpHEV0k3Gi4QYCjzlY82z0AK0PX3mYpJK6fPcGXMCAgJrwzniAAgLJhmdnTRm3RAv1RgZYk7lUies73+woNlgQ14NHrZmbs/H5I24FMRAEATwK4FcBcAPcQ0VzlsD8DWCiE+BqAVwD83+meNxfMU0wD6dycqA6MCCbf3UdtO8OIbiSx2BlTVorGuqqUFo1IVFg+gT7zQeuAe7KdGwKGKivLeSRDUDMiX9wcs8ngJQya2kMxgiArKBKlNxzFfc9uwtk+d9NddxKOwdO9YUfceq6xX5maVR/vePWwDV8d9+1vmzl+pMOvkyjRDgC2d/g3vSTDLo9ky1RY9XGba7i6F/Pi5Jtki0xEE10JYK8Qok0IEQbwEoAl9gOEEO8LIeSo3gQg+8/JywBWJq/J7zYeSOv3LqRq/I3DOJfwyWQwq1xb2atN7SHMnlQe9zt2ojrwro8dn0pENzSOVDUC6/we30+2lHmq9Lk0oK9fR4mHlCtNYkPQF9HRnsQCkg3kBiEdmX0+HPHlOAWAm+ZMdLyuGpk4gu9g6HzCY1JhVIY0DsAYj08n4cDf3tmNx99yNzVmi0wIg6kA7E826TTf8+IBAG9n4LxZR91xvN3snYXphxQKhsZwXtEM5k2pwNMf7kvJHiy/Lye6zF5Ndqd/Kg8x0YWMgDQfOdEIGD08vknPzuK6KtffySXyGeFuDnS/RKPCM+JHRTV5uuVGqGhZCpDvi+e1ToFk52gih3emyWmeARH9BYCFAP7Z4/OHiWgrEW09ceJELpvmimqS6c/Czj5ZQkrm6Qetx7EhiZ25fWkJBgiLzRo2MvOxsS55R6NbctdQp+dCrDlocsVwHD3jfxfbdvIcxo3Kb9/KBXHdnxNvECaNHoavzxoXoxWd6u3HmQuJAyiA2Ig9P5oBZUkLjBelV4xkQhgcAjDN9rrafM8BEd0E4GcAviOEcA2REEKsEkIsFEIsHD9+vNshOUWth5KJnX26qE34bN/JpJ75bP++EAKrzGxVI97d+NSraqYXwQDnLqq47RsOnb7g6Utwo+Vwt6e5KVfIR4V2+PBdyMWzRMk6i1d7SkWN2Nt64FTC7/T4FDTJElSz53LMMY/coWyRiVm8BcAsIppBRKUA7gaw3n4AEV0O4GkYgsCfvlgAFMDan5BzSSwuKlHdaVvXBfD0h/vQUFuZ1MCIFIDG5Jd8iy2vmHg3CN6x67lC+o8m+Qhz7D4fwcd7TsY47pPx36i+hW4XDUslXb+TF5RnYSCAnPoN0p4bQogIgB8D+COALwH8XgjRQkQrieg75mH/DGAUgJeJaDsRrff4uYJCnbd5HhuelJVkbomT8f6UxE8e7vafSJVv8i22kjFDTx4zImH5iWwj/Ud+NAMvklEce/t1xwI4tix/ZjK3Zyjkmlz6DTLiLhdCvAXgLeW95ba/b8rEeXKNnueJ6AcBo1IlMjRwx5aV4PG3vkwq67UQzGfFiN9aWNnkEzMuPpDGTihZP+y67YesApG94eyYgAYLNWNja3hli3xrzQWNOobzbUP0oi+Djq5ZE8vj1q53I1vRHEOd3SkmKmaSw92GwzuXY9++ALoVxxtK3HF57qLweRongVtMeSHgVhMoVeqnVGD+tDFJfWcQuQwGFfn2FwADPo7jZ3PnzLQvgIPJH5UNXvMRxZUpWBgwDpoPd2NkBpNtmMGNNAEmERCUNvaaYJnc6AxGvvBZxyoT8KxnHOw91oOOAtiRMoWB9diKHJ5zm8tzDYYqmU58iwcLA8bB/pPnYhLbGCaXpFrBlkkPNhMxDs5ciGCcj6xPhskWhZDpPxRhYcA4CEd0VLIwYPJIYcbsFT8sDBgHAsBXR/Mf0sgMYVga5AUWBkwMQzt+g8k3pZy4khe41xmGKSjCuYxjZSxYGDAMU1Cw/zg/sDBgGKagKNCqL0UPCwOGYQqKQVAfsihhYcAwDMOwMEjEk+/vRROnxzMMU+RwOYoE/PMfWxHQCL9YMi/fTWEYhskaLAx8ENUFfvraznw3g2GGDE3tIWxSnofMZBcWBgzDFBxL/7/P8t2EIQf7DBiGicv0x/4j301gckBGhAER3UJErUS0l4gec/l8GBGtMT/fTETTM3FehmEYJjOkLQyIKADgSQC3ApgL4B4imqsc9gCAkBDiIgD/CuBX6Z6XYRiGyRyZ0AyuBLBXCNEmhAgDeAnAEuWYJQB+a/79CoAbiYjzDBmGYQqETAiDqQAO2l53mu+5HiOEiADoBlCVgXMzDMMwGaCgHMhE9DARbSWirSdOnMh3cxiGYYYMmRAGhwBMs72uNt9zPYaIggAqAMQEEQshVgkhFgohFo4fPz4DTWMYhmH8kAlhsAXALCKaQUSlAO4GsF45Zj2AH5h/fw/Ae0IILkfFMAxTIKSddCaEiBDRjwH8EUAAwHNCiBYiWglgqxBiPYDfAPgdEe0FcAqGwGAYhmEKhIxkIAsh3gLwlvLectvfFwDclYlzMQzDMJmnoBzIDMMwTH5gYcAwTFxGlQZyfs5Hr63L+TmHOiwMGIaJS/PKW3J+zse+PQd//63ZOT/vUIaFgU/mV1fkuwkMM6RorOO81FzCwiABY8pKcMf8KVj342vy3RSGGRLIRamhtjKv7Rhq8PMMErB9+Tfz3QSGGVIIrlqWF1gzYBimoOB01PzAwoBhGIZhYcAwDMOwMGAYpsAI8qqUF7jbmRjK8pBkxDCS0SNK8t2EIQkLA8ZBUAOGl/CwYPLHvCmc05MPeNYzDiI6T0Ymv3AwUX5gYcDEcLj7Qr6bwAxh6iePzncThiQsDJgYzocj+W4CYzIU86/K2WeQF1gYMA40AFPHjMh3MxiTfJtMpPsol0KJaxLlBxYGjIOSEg2zJpbnuxlMgRDVjf+1oaiiDDFYGDAOgkS4c0F1vpvBFAhSM8lliYhNbV3W3yyEcgcLgzgMCzhH4lAYl8NKOMegUCiE8SZlAOWwMXYzkZ5vO9kQIi1hQERjiegdItpj/h9Tc5aI5hPRRiJqIaIviGhZOufMJYGAs3uGwrisqRyBV7d15rsZjEm+BUKpuSGqGzcyZ+d8p+Wo9Xe+r38oka5m8BiAd4UQswC8a75W6QXwV0KIegC3AHiCiMaked6cUFnmjGoIFODIzHSbjp65MCSE3mBAABie59oMcpd+vj+a8m8kO0b/YBMGI4exppor0h1pSwD81vz7twDuUA8QQuwWQuwx/z4M4DiA8WmeNyeUKSaTEQVoQolmeOWuLCvlpLMCYURQw9jyYXltw+HT5wEA3ef7U/6N4UnOm1vqJ6X8XSZ10hUGE4UQR8y/jwKYGO9gIroSQCmAfWmeNyccNCeCRM9TOxIxeXRqC0bA5e4vqK1E8+HuNFvEuBHUKCmzR78uMLVieNba4wcpBKaNLUv5N0YkUetq/KhSPPbtOdbrfGtG+aY8h5pRwp4mog1E1Ozyb4n9OCGEQByzOhFNBvA7AP9JCOG6rhLRw0S0lYi2njhxIslLyTzTlHj7QhyWAQJ6Iymq8C53684F1WynTQIvE4jb21fNrMLEJAR3iUZ5N9nJ81dXpi4MLq+pxKzx/nwOJ86G0dQesl5PGeI5Lz19qZvnkiXh+iaEuEkIMc/l3+sAjpmLvFzsj7v9BhGNBvAfAH4mhNgU51yrhBALhRALx4/PvyVpkjIQSwuwgNuo4UGcu5DagFFNTGNGBNFQW5l0aGkh+lJyhVfIpdvbH+05icqyUt+/PbxUwxFFO801cmeeqlAKEPDodTPR0+c/q90eWppvYTiUSHd1Ww/gB+bfPwDwunoAEZUCeA3AvwshXknzfDlFrZHy/YZpeWqJNwSCyNCUOXMhgqb2EBpqKzFnkv/Es/Gj8mvXzifJ9nyoN+z72L5+HWeTWESzQflwI4iiO4l2A0YoKsGIyGs92oNjZ/p8fU8jZ2hpqDd1X0WhUXhbSSfptu9xADcT0R4AN5mvQUQLiehZ85jvA7gWwP1EtN38Nz/N8+aEM3meiH7QCBgezIxdURfGrqypPYSvjvb4/t7Jc2EMG6K23dFlQdf3vbSlZEotaETQshDgP2dSOS6r9hckUGre11PnkhMGQhiCsj+iY82WDt9C88Y5E9FQOxChPiOHIa3ZJpUtWy617rRmsBCiSwhxoxBilmlOOmW+v1UI8aD59/8SQpQIIebb/m3PQNuzzp9ttksA+P3WgzlvQ6IMzO8vnIb6KZmp8kgwFqtNbV0xAzdeM3QhsGjG2Iy0YbDxD9+ag3/67qUx98ltDb9j/hQrOscPOoCaNBy3XiyorcTy2+t9HSsX4xnjR6V0LgEgHPEfenHD7AlxXw9mSlJY2UvcojyyxNDczvlEVekDeciNjxdnTQBurp+EBTUxuX4pUVVeiobayhi7NiF+BmrVyNJBFQJov42pbrwvmjAK//TdS3HvohrMnlQeIyx1Zf27dtY4PHH35eg41ev7HCNLAzh6JvPlxOunVPhOLNxibogevW5myufrj/oXBmok2wetrm5IB7malhoBUytTd2iPTcJfJLmQhCBNFxYGcVBV+myqrKVBzdVOP3Occ0dmP0YAWLutExttDrd0OGVGcoR6w44JtnB6JUqDGgJkDJhSZYfzf9w8G5v3u7eh0HzLk8qHWUJdI+CK2tQE6cjSAO5dVIOm9hDueWZTwnyPfSfPAUjOTHR5TSXGjkx+AYmHBmOTc6LHnw1fljNvqK1M2WRRl4RWoZ7imA9hmK2SFRoN7OYDGuGXd1yK6y9OPbDlmM8+tzNozETFjlq986IsVfO8aMIovPhQI65zGWj7Tpx1vC4f7rRRE5Cyvd5tnG1q60JjXRWCNmnw54Onsfy2evztN2fjl9+9FH9z08W4Y/4UTK8qw6PX1uHeRTU4c97dvzIpxRyIdPHS4o6f7UO/uXLrAhhTVprSA9jbu3rR1B7Cq9s6Xc0gQjm9XFSTqQh7/ewJOJfh0MLSEg2NdVW+7dfXzhoYk5VJCKbasWX4+qxx+KfvXopHrpvp2cd24RzQYosk5tNnUDduJB64egamV5XhoWtmYPakcuw+5t+XZkej1HwGuazNxMIgDuoubmmWqnmONJNynvqoLeYzUuwYfREdpUENBEObuHNBdcolpwMBciSeBQPGQtFQW4nrbbbaSFSg+XA3GuuqsPLNFvzLH1uxbvthHOjqxXOfHcDjb33pOtDvmD8Fl1aPSalt6VAa1HD71ybDLRJYnVwCwMollyad3HP6fD/uXrXRc4ddoggjmdXtN7SUYOzgj3RnLrS0esxwLL+tHg21lZjgM7PZPraSqU90zaxx+N0Di3Dvoho01FZizSNX4ea5E3HR+JEOrfNoTx+i5k2J6gKtSuBCV5KO60xy9kIET33UhgNdvXjqozbcvWojthwIJfxeQHOarr45dyIWpqiB5tIyzcLAg1yaNxbXVWGthw333itrHK+XXVGDFx9qxN99azZefKgx6bwAsv1fP3m0FSdPAL7XUG1FcoxXFguCoTWEI7pj4Q9HdKzbfsj1XPl4LsKV0yvxw6umY932w+j3YW7t7g1j+evNKSX39Ee9g3qDiuNv15EzAPyHlgY0QmNdFSoy+NSvztMXsOKNFqze7C+6Rw3zTIQMJ5WbFDsNtZV49LqZ+O6C6rhmo+c+cW6Ibp03OeF5Rw93j+hKl7PKE//6fdZ+uXiC04f0we7UE2iHJ5G9nS4sDDwQcCa/wOV1pigfUeIqfOZXV+Cxb8/Bo9fWOUwyDbWV+NENF1kLd0NtJeZOTrzwzq+uwLASw/Y/rETDsitqLF/AsBLNofncuaA6RgNprKuyQg3teO129xzr8eUAzCT/cOsctJgLrxclAaMsRDBAaOo4jUiKunhJgDChfJjr7k219Z48G8bqzR2xznkynMsqD14zAw21lb61UVUT8SIc0fHz15vx4uaOmM+mVo5wXItqajuVIOZ/yWVTcM2scVhxe70jPBQAmtpDuO/ZTfj1n1px4ORZj18ALigF8e5dVIOLJsT3OSSTyAcAZT6TR8cogjjgs5zIl0d7HD6kcETHoRSTByOZLj4Wh6ITBpm8IHWQNdZVZUVjaKyrMspAKD9ePqIETe0hPL/xADpO9eL5jQccqfp2EpULuGjCKPz89nosv60eV100Dstvq8e9i2ocr+0TuKG2Eitur3dM7obaSiy/rT7GD7CgthJXTo9Vg9dtP+y5m/Lbj6OHB11/24u12zoTPlD9+tkTcM2scfjG7AnQUxQE1WOG46WHF1tC0w9vNx+JiZYRAvj8wCl8c+5AWS+NBp4D7Pd5wBOSqGEU1d01mnuvrME9V9ZY90bXhWMD1J8gsuXNL47g070nsfLNlphxKrVKXRiRVl495lZ+Yl6C0OmoGrqVgD6fC+zZ8IBgIgDLrpiGm+bGLb/myZHu1KLCcmmhyI5+lUcyGYilqvQNtZWYOHoYjvrMpvTDnEnl1iK85LIpWLf9sPVZ1chSPLFhtzWJ+iM6NrV1ue663kuwA993/CzuWbURIEIkqmPLgVMAgJVvtqCvX8fGfcakv3dRjfWbK99sQTgycGzz4W680tRpLQoEoMRmErjrqc9cbfJ2Ahpw9xXGOV5w2Z2qNNZVoaw0ACCxrRYAXmnqjBvxEQwQPmg9jkhUIBAglAQ1RCI6NI3w4DUz8MYXh3HodOzE1WggkaokQPi3exZY9+GFBxvx1If78M6uY9bxcyaPxueKfbl+8mhs64i9jr5+w/QW1AhRXUAjsjYifs00UyqG41Ao8e4zQEZWcCSix8yVyrJSNNZV4WXzHgdMH5IkkYlLCpmwyziVWmV/RAcRQTc7U4PRp3KcbO/strLgAWD15g7HnHDDT/ilXFQ1guWjSMTXplbg8wOnjL4wtQK/vpYYUtzgB3PoNCg6YZAp3OylTe2hjAoCAJhtCxVVbexvfnHEmmCEATuyyqvbOhOqkwJAOCqs4hX9ER1vNx+xFqKILrD89WbMNoWTfScX7tex/PVmx45SA3D1rHH4yU0Xo6G2Eqs3d7hGPmgU67S9c0F1jKPQiz/ZFlg3aseWoeNUr9Wu/ojuGY44flQpLq+ptH4zEhX45twJGFc+zFos7IIgoAENNZXoi+ioGFGCj/ecBGDsmFuP9jjMdM/81UKs3tyBt5uPWHZuVRiUjyhBn1vkEYB3vzxm9VNEF1ixfuBezJ1cjl1HvPtLI+DiieVoag8ljD4JBDSsuL0eod4wdhw8bfWFDDltPdqDqMwLUAovxdOA5ldXYHunofXoIlarllrl281HUD95NJ77dD/6owLBAOH62RPwzq5jEACiUacgebv5iHqqWBKsl0HNCBII9YbRc77fNVDDjUV1Vfibmy7GUx/uw3tfHceLn3cgqBlBF16pE6OHBzF/2hh8ZI4Vqw1BDfOrK2LGRCL6ksjRSJeiMxNlCs1FIqfjMyACHr22DmOUB+as234YP3ttJ5raQzETyL74CvkjLiSz6SgxfQQlQQ23zpvssAvrYsAsIHdyATL6QhdO00IwqFmCAACe+3S/e9uUxkV14OkP9yHUG86ICjxx9DBHuwQMh3zQJUDbLTJFwBCmL37egaeVRWLS6OHY1nEaOzq78dGek9Z5ogJY/npzjCnk3kU1VgTN+4qmJjcXy65wBgRI1EU8HBVYu60Tqzd3xBUE8rvlw4IoDWrWhCa4r5ERszzE4dPncf3sCRhu+pBKSzRUlpXi5+t2ImpqQBHFTCQ8qvIRgC86neYv9fqlpvnp3pN47tP9iJrjSYdhtpO+rJKgUxtxcyCPLRvwsZUECMM9snQJwNdnjcOaR67CvYtq8KMbLkpYMK8kQAgQMNwMwW092oN3vzyGqC6MjVFUoHqMt0n2G5dMwKf7YteJ7zVUpxSansvQ0qLTDIiSf3i3Rkb0R78tUkaPihhVN1lHlZ0ll01B+YgSXH/x+Bi194XNHXh560HctXAaCAOLe8A0G1iLUNTdTLR0QTVW+zC5zJlUjl9+91Irl0D+zvLXm6ELgVLbRGyorcQLDzZiU1sXKstKsWJ9M8I27UNXO9mj0wMaxThot7aHrAVAaiap4lYz50xfxKzp4/xlIYBx5cNQGiD0R4XlAJYakMqR7guek1EKTvu9aGoPWX27X8kPGTuy1PK7/K9NBxIu8ICxmPnaGQPY2NbluF+h3jAqy0rxj+t2WtdAMBbfHZ3d2NHZjdLggJbQaEa02RVMogFNtKk9hK5z7g5kt9r16vW/uq3Tutf2cSTDlmXb7eMSMARsR9c5PP1Rm3WOUG8/NDLqGD1y3Uw8/eE+dLqY9koC5NiwNLWH8HKckjIagLsWTsPUMSOs6zbmhvO49jhZ5K9vPxzTFxoZocWzJ5Xj91s6kExS8Ygc1vwqOmGQyspSN34Ufnj1DLzfetyy++qIXfz9hwUCv1hyKV77cycOnupFY10V/tByFOt3HPYsPBaOCuw+1uNo/m1fm4yyYUG80tSJaFSP2TVJGkwHbiIVdIG5GAEDWo4sp+A2EeXi1dQeiunWSFRg5RstWG46l394TR1++trOmHPe9rXJaD7Ujb0nzlnvnToXxso3W7D8tnpLdfey1Seibvwo7O86Z6ntMgIq4qJeBwKEpQuqsXRBtXW9rUd7oBFBiFinarxdWalyL2S0TNjMA5lR5YzJP3XOyO5uPdrjEAQEYI6LGUhGcLUe7bHMU/E4eS6MV7d1on5KhbW4v9Ny1HENMyeMwr7jZx0mtebD3ZhqOm3VkXnjJRNixotfxo4sxZPv77X66OWtBwdMjIrpkDAw1ty4uX4SnvvsgJXcJ2BoZ+99dRyPXDcTj1w309WcKAC0Hu2x7vWmtq644aGlZkSdbMeT7+/17V+wn1NFF4Zv7oUHG7Hmkavwj6/txJc+zaSTcvhwo6ITBqnsMvceP4sVb7TgroZqa6BqFLv4G5m5SCjZZaLYjoOn0R8VeGPHYeim6h2zo7YRjugOzeDNL45gzSOLHYuX14T5h1vnYNnTn7m2ze7oVRetFx5sjDsRAWNX5zaJdnR2475nN+GFBxstx/Mv3mzBeVuA//6T5/Cr710W07b+iI5Qbxg/uuEiNLWH8Mwn7mYmwHD6Sp9IMEAQwjChlZj25ve+Mu3eBKy4vR6zJ5VjrW0nKpFdbxdyK99sMZ228v54NsM4B4BLqyuw7IoaR5/ZfSz9Zt/aEcI4Rl1Uv1ZdgeW31+OeZzZZi93YkaX4vpnzIc+xZksHzoWjjsXczqHQecshr5HheOxXLmZkaQAlAbJ25oEAYc2WDkT1Abu6dPKWBDU8YqtH1FhXBQ3eARpXTK/Eto7TVl9u7+zG1vYQSoPGAiu1Q4Kxo/9w9wnrPPVTKizB4TYON7V1uQr3qGnGaqyrQoBin8/RHxX4uakZlQQIP7x6huf6cJl5H1Sn97ASDWFzPNtbcMf8Keg6F0b95NEOrQVw95PJ4I/GuqqksphvmpNa9FIqsM/AJBzRcaKnz7KTqzs/wFhEpo9zj3kuDQ7sqyJRgec+aUPYTEqSNthELK6rcrXhq3kFbtizPO0QjAVHhoeqi5afHV+8tof7dYeW8fVZzkieiaOHW227d9FAXkNJ0LBRP/n+Xjz94T7PHRjBqMx676Ia3LeoBiu/Mw8BGrCJf9B63BIyujAinqSJS32qWFSxgatJdDfOmRg3eqN8eBCBAGHnoe6Y8Em7j6UkqGGxMnaCAcPkotrApVBZcXu9NRlPnQvjqY/aLNPfvYtq8PqPr8Gvln4tJvbfrbW6MBPilC6dMW4kXnx4MW6eOxGXVVdg1vhRlkYV0Q07v5rQKGmorcTD19Y5fi+gmUlmAcJ3L6+G/blskejAGBOA5c8IaISZ40biuovH42vVFfjhVdOx8s0W/PpPrbjv2U2uodOyb9VrlQEVm9q6XGtDaTQw98JRgY1tXVYehQzIkHk0qiCQ1/zCg434r9+ajZf/81X4p+9eapXYeOLuy/G7BxahpmqkY34QgLuvrMG9i2rwzbkTHWMiXlu98BtanAmKTjMote18kmVc+TBP26WkbtxI7D0emzRTVhJAODLgnEr2AeIytnzlknnWbiboET3kRUNtJeZPG+MIcRQwnHutx1owe1K5I8TPy+yksnRBNV7ZehD9UYGABiyoGTBJqea062dPcKjssqyF3OVKLaeyrNQKXY0nbQIaxajuEdOPEtEF2k6ecxx/0iwP0VBbiZqxZY7oLzVCTO2LR6+biRtmT7CcqCpnL0QcJha7z8DuY5H2dzvXz57g2OnLqCOpUYV6wzG77rebj1ify3OsXDLP4QcAELMrtmsGdoEgHegf7zlhCEHlGo+fuRBXS5TJkQLGLnLZFTWWfX3ttk6HUA5ohAAESkzNYN6UCisizR7N03z4DIQQcUOnZSSS/b5oBKxcMs86VvUQVY8ZjrlTKhxj8VxfBMGAZplcpZkynsZt74+G2krH/QAMjc2OgOEfsIdoq+uJ3zUq2QzwdCk6YZCKGCAYaqRcdOLtwB+5biY22MIAJb1K7YNSpaRzwAxULwlquHL6WHxxqNsRx2xfmAMBDXpEt6KH3AaUFz0uQkhgYKL96IaLEgo8lYbaSrz48GKH7XWrGcaomtNkxVMvU5vs3yff3+vLefzgNTMAwDIjVJaVWn2vCyOyxM44Wxz4GMXns1C5t+oCDhgLsFeb5PvS7BZvoqq7WHt8+r2LamIWFTcTpKpFyIqyD3+9Ds9+st9y+i+/rR7Nh7uNEiOKz8C+8I4oCViF9XQR20avSCd7G60ADYJDSD/94T7HsTdeMgGXTRtjjbFNbV2uyW5RXSCoGc/ri9enod5wjHPbHpatBo5UjRqG62dPwAe7T1jmt7aT5xDUCHdfWYM7bW1PBTkn3YpErnyzxQoLVtcTOZfWbuvEGzsOo+eCd3RTsoEw6VJ0wiCapFYwqXwY/vKq6UktjL+841L87LWdMTV6JMEAOWLdNQIeumYGykeUOHbEnx84FbM7efL9vYhEdSvmeu22TmsC2238bjz+1pcx4ZF2IWSPFEp2Iqjf8dIu/GoelWWlCQUBwYgMsvs41PIMFcqCLwvCAU7BALhXnbX7D+57dpMloKwdsLnL1jFQGkDTKCZjW5aylte94vZ6Rz8kqh8lTWlPfbgPx89cwLIrnAJD9fWsXDIv4a4WAJ75ZL9lgtvw5TEENXLsju9fPB0tR844tBQvfrfxgEMQ/27jAavv7GVHSgKER66bGROJ53a/1Ygmr2txC7t+YsNu/OSmi42HMSk/vvOQoQ2vuN3Ibfh070nopq9pypgRKQkCKQDsczioUYyPwG5+jReY8ef2UFxHsjC/n47QSoaiEwbJpmgc7enzLQgkctL83FR77YwaFsBVM8dhw5cD6qkugOc3HrB2oVYyl82JKlEXUwJ8ZSCrGbCAsYj5XTSSQd1Rx9tte51T5hnEEwgChtnH3l+q8607jiZiFwxur+3Y/Qcyoe7WeZOtHfeeYz0D0VpCxGg89lLW4YiOlsPdePGh5DQwAJhv2027tU+OA3XcuPHqtk7H+JSL4bIrB8InkxkTasG1DV8ew5Pv78Xh0+cdDuK7Fk6L+V31fl9WXYF5Uyt879Ddxsune09ii7mhko5xqe3Y++knN12MLbZM4sOnz1u+CXsobrz+sAtjwoBZLhIVjvQfzeYPcwvUsGv5fh76k044e7IUnTBItMC48dSH+/DMXy1M6jtSIPz3t3eh58JADZOzfVG891WsGUku5KqZo+d8vyOSws10sXZbp+dOWw7SCy4lOr9xyQRLXc008bQLt89UU5dfzcAe4aOL2Aeky5Lebv2TyGRlRxXCP7npYgAD5TrsbVXLNACxY0549IMXXlFeXu3zY0tW2yTNW0sTLMBeZsnpY8uwvbfben0uHMWv/9Qao224FddT77eq+cSjqT2EQ6fPW6VDYJqE5ILffLjbNKkaC3NAI+j6gDYs59TabZ14pclIMHx560GAyHJwa2bQiJfmbRfGjj7VyErG0wBcfdE4S1txC9SQ9zioka/SFn7D2TNBWsKAiMYCWANgOoADAL4vhIgNBzCOHQ1gF4B1Qogfp3PeeGhxUsW9OJ7CowWb2kMxiVgS9fx2G/Omti5rkSPAYfu1h3naB2S8nbYcdG5s+PIYPtpzIq5pKZN4LSJN7SHcs2qjlej14sOLHTs9gjGX3QKKJpilIuSOfYbiwF92hXeeRDILqJtG8+T7e2NKdhOcpb4ldie79D8lg9viIc8h+9WPw9OOqgndPHdijPlGJZ5QUiNbZK0mP9pGi61AnwYj6iteOKlbe4Ia4Z5FNaifUoGVb7Y4tGdZL0sXAOkixi8g/RYyysmYt06tyUvzBgbGkn1jQDA2XB/vOeHYRFiOYmXsOawCUeGaKKcymDSDxwC8K4R4nIgeM1//g8exvwDwUZrnS8jI0iDOxHHKuKGGAfrBLYFFLmrSxhw1o2+WXeEcmMNKnAW7Eg3EeDtMOUjdNINEv5tJ7Db3gGmekju/tds6LaEpyywsXVBt9YOM7Gg+3I2XPh+ocSRgRIDYj5MRP2o0jle/JeMsV/tZ9m243yjqJnePbgu96mT309924ekluBJpDPGw7yoJwGXTxiT8bjyhdOu8yY4EuJIAIRoVICJHBI3bddozf7UA4ZWmTkSiia9pU1vXQP2sqGHvVxMlAWDNloOWqUoXcH3WtHyCn30DZ/cN2cOdvcyfUruQWtCj183Eo9fNjLnvXmNPFSiJGDSaAYAlAK43//4tgA/gIgyIqAHARAB/AJCcPSZJvnHJhIRVDlVSieVtrKtyJPAAclAZyS3PfXYAUQhomhazQ7EX7Hp+44GkVH8VOege+O0WnFZMKHKA5yI8zTFplaJ3atQKwXuyvPflMUc46PaDp2OOcwvx8yIVZ7n9u2qJB79hiIlwW+Td+iPe4pwI+65SwN8uM542JUtD/KHlKG6pn4SaqpFWKRN7BI3KprYuh0+hfvJo7DzU7eua7OYlexiz2tcrl8xzFFOU/gS7oGmorcRdC6dZD/ex+4Zk+Q7pGHYTUvKcSxdUY+22zoS1tdwiiewCxUujV68/V6QrDCYKIWTxlKMwFnwHRKQB+DWAvwBwU5rnS0jZsOQvacfB046yuX6QO8GVb7Rgh61Il64LtBw544gIUlV+e2noZFV/r7aoNY9mjR+JSWNG4NZ5k3NiImo0E+YGdmcDCXN3Lqi2yiLbI2vcFs875k91hEPeUj8p5rhkQm3TJR1hEg+3Rd4tsTAVX4EkGZ+JJJ42JZ+tEY7oeH7jASxdUO1Ls7VrWJpGWFxXhdZjPb6uye81zJ5Uju9fMQ0th7rjCpo7zYXczawjzYJ+hJQMGJC+Bz9ajuxfKVD+8jeb0BuOLxAKSjMgog0AJrl89DP7CyGEICI37eevAbwlhOhUn+frcq6HATwMADU1/nZ+KntTeGD1n3albluvn1phLv7GpctqoFtc8geA1KJC/DBrYrnDeb6/qxf7Tp7DlgOnsuJEVhfkhtpKa3fmVvTOK7JG/Z2b6ydZ4ZABjXBz/aSY86ZqNkn3GjMphNwWebffT9bUlegcfvASgOrYFfAOMVZ/b/lt9dbYeH7jAd+bID/XYPdJBTQ4nNluVQTU/rSHjPq5Hkc/mL4HgeQ0NxlE0Rv29hsQCizpTAjhuZsnomNENFkIcYSIJgNwe8LKYgBfJ6K/BjAKQCkRnRVCPOZyrlUAVgHAwoULU0q5cKti6YdkbmRTe8hS9SJRw7H1zbkTMb58mGUSkvVxVPGXzk4vHrKOivRFWA8a6c+8z8BrQfZT9C7R7xgx49JpEFsZNB2zSTKs3tzhEGzLb6uPa0JIFnVRAuAp5FLVTuwmyUxoiOrYVYv+xfv9UG/YoUX43QT5EYZ2n1REB26+ZLxniK78Tbumbu93P0LK3g8BjQCiuIUkvaifUhG3OGOOc87SNhOtB/ADAI+b/7+uHiCEuE/+TUT3A1joJggyxdiRpcCJc4kPtJGMbV1NTgKMaIrLpo2JGdxSlVy7rdMRKSTtholsjslgnzT2B3i4VV91u6Zkdp7xFuRkFi6330kkLBvrqqxy424hnpmgqT2E5a83WyavcMR4EFCmhZC9r5IxUSRzHSvWN6M/KrC5rSttDdFrYfbzm+lsghKNKXUeTSgf5lvbTkVTdxPkqWhuaukWr/blwswLpC8MHgfweyJ6AEA7gO8DABEtBPCoEOLBNH8/adQSBImQj2H0m/yiFjczfiO2hlCiHayboEgXOWmefH+vb1txKmaXTGk3br+jOm1lfLajTTbNIRvI0gkSjSiu6S8TZENjVKO47CXHUyUdLSVVc1civHxSfsiUKS2V60nkDxhUtYmEEF0AbnR5fyuAGEEghHgewPPpnDMRagmCRAiBpNLT1XBDAK5PIIs3yLJp6mhqD2H7wdMgImhw2u7dSKUtmZrYiXaabkJKRqbI+PZs7JykyU06PGWYrJcJLBMk26d+tDl1VNpLjudqt2knU854N39VKtnesk3ZElKJaKyrci13LfnOZVNy2p6iy0BeuqAaaz7v8F0mVqPkK4O+8GAjntiwG5+Yj0KMuDyBLN4giyco0nFSSkeaVa/epYaOSqadjMni11kp+zdbPhe1TV5CKlOT0/68ZHuuhF+flR9tTu6Y7SGMuco7yRbxnseRzqYk3nezGb1mz3uoLAvidO9AZdy3mo/iL5OMckyrLTk5Sw5pqK3EQ1+v8/3Q6wevmZF0ZzfUVjoScHThbpf3GmRei026kTJqIpyux9bQ8duWdHBb6BJhn3AAcOj0edeokGzsoN3I5MKvsnpzh/VUODmG/PYT4F+bkztmNVEql6aHTJOrAAJJNqPX7PkXAQLmTR2DT2xJfbkW3EUnDADgzS/8J50lekC2F8229HrAqGuejIPObbFJd6CriXB+J36md7zJLnRqyQEZt+1VbjjTO+hcoz7XWH1uQSKSLbPRUFvpO+qn0MmFZmgnm8JHvZZb503G5raupOdvpig6YdDUHvJV80OSqgtSPkRFkgl7bLoDvaF2oFY6AWnXbE+FVBY6r7jtSFS4lhXwi596P/lYHNWyDupzCxKRijaXTU0nl2RKk/V7/7MpfNyuxR6Snuv5W3TCQEaf+CGgIemCYhI3R3W6Mf2ZGOj5nvSpLHRucduRiOGgdysr4BeviZxvjUEKx2RNaRK7Ga4YFvhkSXeMx6uj5XaubDqY7dciBVSiqrLZouiEQWOd+8Ox3dASZETHw602vp+Y/kTkezFPl1QWOnXCAcATG3ZbDyRJVT33msh+Vf9sag9uTzvzQ7r+hmIlmXsVr46WG9mckzKB9WRPHz7YfcJ3WYtsUHTCoKG2EjfOmZgwmQMwzBCp7uTfb41NtibktpaISj5NH3ZSWejUCWd/IEk66rnbRPZb4qAY/Q3FiNe98poP8epo5brdy57+zPGoUyB/EV9FJwzUR/DFQ9Mo5Z282zMQ/FaGzAaFunilw50LqrNiO/Wj+uc6asUv6fobihG3ewXEL+/hVUcrl6zd1hkjCAi5dxxLik4YqOGVBGDO5HLsOuIsYGc8QDt+6d14LLuiBjs6dzre05A/zcA+IcL9uvV82EJYwJJFFWzJZJT6JZHqn+uoFb+k62/IJ9nSXN3uVSJhnu0kQj+oRmqNgHtcoudyRdEJA5nVJ+WBAGIEgfxAR+q7Plnbfd32QzhxNhzz0Plco2ZGp+N4zSdN7SE8sWF33nfl2XYcpkOq/oZMkOqCnk3N1eteFaIwt2NPCgxohF/EcWTngqITBn59BvanG6UyUOy13TUySlkvu6Imb4uGnBCZcLzmA3slWPtzafM5kQe7Mz/TpLOgZ9vspt6rRMK8EMyqMimwUDYcRScMAOCR62bi3a+OO4qNqRAGHl6dyk2wD25dAF90dqP1WGomp0zRUFuZMcdrLnGrBGt/uHi+JwljkM6Cng+zWzxhXig+oULacBSlMGiorcQvlMfgqQQ0SitOu7Ks1FFgSn24Rb4iewrZvOGFWgmWAJSWaCwICox0y1AX0rj0ey2FEqGXC0hkqQxwuixcuFBs3bo1rd+Q9md79AVgOI8JSEs9fPL9vfiXP7Y6FrBhJcbvAd6RDEwsUjOQSWd3LZyWMSfaUJrMuaCY+jPRteTKlJTpPiWiJiFE0s+aL0rNABjo4FvnTcbGfSedIVxpOo+BgTpA8lF7y2zPRMjGg0qKmWztGgvBLlxsFJJZI10SXUsuTEmFNEaLUhioHfyNS5wOZU2jzET/EAEQ0DTNsZMt1LDEQiYbi0yh2IWZwUku5nEhjdGiFAZqB48rH4ZSWzVPIudOPtVzRKKGnTuqPM+g0OyjQxUWykw65GIeF9IYLUphoHbwUjOT9YXNHQAAPSqSerqZn3OoN7GY1OnBCgtlJl2yPY8LaYwWpTAAYksZtB4dSDzTAew4eBpNaTxFqJBuIuMNC2Wm0CmUMZqWMCCisQDWAJgO4ACA7wshQi7H1QB4FsA0GFGY3xZCHEjn3F54lTII9YYdzxt9Z9cxfLTnRFoOm0K5iZJiivRgGCa3aGl+/zEA7wohZgF413ztxr8D+GchxBwAVwLwV0kuBbyKVkmzjqwHYs8LKAakEPz1n1px37Ob0NQeI5MZhmE8SVcYLAHwW/Pv3wK4Qz2AiOYCCAoh3gEAIcRZIUTqj69KgFz0A0opA2nWuWdRjevngx0vIcgwDOOHdH0GE4UQssD6UQATXY65GMBpInoVwAwAGwA8JoSIpnluV+y2/MqyUmtRlCadhtrieR6snUKKSmAYZvCRUBgQ0QYAk1w++pn9hRBCEJFbOnMQwNcBXA6gA4aP4X4Av3E518MAHgaAmprUq/fJBT5ePfNiEQKSYnJos++DYXJPQmEghLjJ6zMiOkZEk4UQR4hoMtx9AZ0Atgsh2szvrAPQCBdhIIRYBWAVYJSj8HUFHhRTfX+/FIOQK6SMTIYZSqTrM1gP4Afm3z8A8LrLMVsAjCGi8ebrbwDYleZ5EyLNJhpg1fdnx2rhw74PhskP6QqDxwHcTER7ANxkvgYRLSSiZwHA9A38HYB3iWgnjJpuz6R53oRIs8nVs8ZZIaW8uBQ+XgEADMNkl6KuWgoYZod7Vm1Ef1SgJEB48eHFbHYocNhnwDCpw1VL42EWlDP+ZwqdYvB9MMxgI10zUcHjVlCOYRiGcVL0woBt0AzDMIkpejNRMcXfMwzDZIuiFwYA26AZhmESUfRmImZw0dQewpPv7+V8EIbJMUNCM2AGB5x9zDD5Y8hoBrzjLHw4+5hh8seQ0Ax4xzk44MqrDJM/hoQwcNtxsjAoPDjyi2Hyx5AQBrzjHDxw5BfD5IchIQx4x8kwDBOfISEMAN5xMgzDxGPIRBMxDMMw3rAwYBiGYVgYMAzDMCwMGIZhGLAwYBiGYcDCgGEYhkEBPwOZiE4AaE/jJ8YBOJmh5uQKbnNu4DbnBm5zblDbXCuEGJ/sjxSsMEgXItqaykOh8wm3OTdwm3MDtzk3ZKrNbCZiGIZhWBgwDMMwxS0MVuW7ASnAbc4N3ObcwG3ODRlpc9H6DBiGYRj/FLNmwDAMw/iEhQHDMAwz+IQBEd1CRK1EtJeIHnP5fBgRrTE/30xE022f/Tfz/VYi+lYBtflviWgXEX1BRO8SUa3tsygRbTf/rS+gNt9PRCdsbXvQ9tkPiGiP+e8HBdTmf7W1dzcRnbZ9lq9+fo6IjhNRs8fnRET/w7ymL4hoge2zfPVzojbfZ7Z1JxF9RkSX2T47YL6/nYi2FlCbryeibtsYWG77LO64ymOb/97W3mZzDI81P0u+n4UQg+YfgACAfQDqAJQC2AFgrnLMXwN4yvz7bgBrzL/nmscPAzDD/J1AgbT5BgBl5t//WbbZfH22QPv5fgD/0+W7YwG0mf9Xmn9XFkKbleP/C4Dn8tnP5nmvBbAAQLPH598G8DYAAtAIYHM++9lnm6+SbQFwq2yz+foAgHEF2M/XA3gz3XGVyzYrx94O4L10+nmwaQZXAtgrhGgTQoQBvARgiXLMEgC/Nf9+BcCNRETm+y8JIfqEEPsB7DV/L+9tFkK8L4ToNV9uAlCdg3bFw08/e/EtAO8IIU4JIUIA3gFwS5baaSfZNt8D4MUctCsuQoiPAJyKc8gSAP8uDDYBGENEk5G/fk7YZiHEZ2abgMIYz3762Yt05kJaJNnmtMfzYBMGUwEctL3uNN9zPUYIEQHQDaDK53ezQbLnfQDGTlAynIi2EtEmIrojC+1zw2+bl5rmgFeIaFqS3800vs9rmuFmAHjP9nY++tkPXteVr35OFnU8CwB/IqImIno4T23yYjER7SCit4mo3nyv4PuZiMpgbATW2t5Oup+HzGMvBwNE9BcAFgK4zvZ2rRDiEBHVAXiPiHYKIfblp4UO3gDwohCij4gegaGNfSPPbfLL3QBeEUJEbe8Vaj8PWojoBhjC4Brb29eY/TwBwDtE9JW5A84322CMgbNE9G0A6wDMym+TfHM7gE+FEHYtIul+HmyawSEA02yvq833XI8hoiCACgBdPr+bDXydl4huAvAzAN8RQvTJ94UQh8z/2wB8AODybDbWJGGbhRBdtnY+C6DB73ezRDLnvRuKSp2nfvaD13Xlq599QURfgzEulgghuuT7tn4+DuA15MZUmxAhxBkhxFnz77cAlBDROBR4P5vEG8/++zkXjpAMOlSCMBxlMzDgzKlXjvkRnA7k35t/18PpQG5DbhzIftp8OQwn1Szl/UoAw8y/xwHYgxw4r3y2ebLt7+8C2GT+PRbAfrPtlebfYwuhzeZxl8BwrlG++9l2/unwdmz+b3A6kD/PZz/7bHMNDJ/cVcr7IwGU2/7+DMAtBdLmSXJMwFg4O8w+9zWu8tFm8/MKGH6Fken2c04uKMOd820Au83F82fmeyth7KgBYDiAl83B+DmAOtt3f2Z+rxXArQXU5g0AjgHYbv5bb75/FYCd5gDcCeCBAmrzfwfQYrbtfQCX2L77Q7P/9wL4T4XSZvP1CgCPK9/LZz+/COAIgH4Y9ugHADwK4FHzcwLwpHlNOwEsLIB+TtTmZwGEbON5q/l+ndnHO8yx87MCavOPbeN5E2yCzG1cFUKbzWPuhxEYY/9eSv3M5SgYhmGYQeczYBiGYbIACwOGYRiGhQHDMAzDwoBhGIYBCwOGYZiCIVFxOpfjv09GkcsWIlqd1rk5mohhGKYwIKJrAZyFUY9qXoJjZwH4PYBvCCFCRDRBGElmKcGaAcMwTIEgXIrTEdFMIvqDWWfoYyK6xPzoIQBPCrMoYDqCAGBhwDAMU+isAvBfhBANAP4OwP9rvn8xgIuJ6FOzwGJaVWu5UB3DMEyBQkSjYGTIv2xU4gdglNQBjPV7FoxnMVQD+IiILhVCnE7lXCwMGIZhChcNwGkhxHyXzzphPDioH8B+ItoNQzhsSfVEDMMwTAEihDgDY6G/C7AegyofI7oOhlYAs8LqxTCK6qUECwOGYZgCgYheBLARwGwi6iSiBwDcB+ABIpKF5+ST1v4IoIuIdsEoFvn3wlYuPOlzc2gpwzAMw5oBwzAMw8KAYRiGYWHAMAzDgIUBwzAMAxYGDMMwDFgYMAzDMGBhwDAMwwD4/wHlQQF6Kvi4MAAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(torch.arange(0,len(conc2)), conc2, '.')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[<matplotlib.lines.Line2D at 0x7ff7759c01f0>]"
+ ]
+ },
+ "execution_count": 61,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAdGElEQVR4nO3dfZRV9X3v8fdnhgdFjQw4UgIokBAbSG9RTxCbrNbGBpHmFpumCYmr0lxzSXrNuvEm6zaQ9ObBNGuZ2yamrpsYqNqQLA1yNVFqSA1RkrTrCnJGUZ5CGBFkEGGE4UFRYGa+94/zm/EwzJw5A/PE2Z/XWmfN3t/fb5/925vNd+/z20+KCMzMLBuqBroBZmbWf5z0zcwyxEnfzCxDnPTNzDLESd/MLEOGDHQDSrnoooti4sSJA90MM7OzSl1d3SsRUdtZ2aBO+hMnTiSfzw90M8zMziqSdnZV5u4dM7MMKTvpS6qW9IykR9P4JElrJdVLekDSsBQfnsbrU/nEou9YlOJbJV3X60tjZmYl9eRI/zPAlqLxbwB3RMTbgSbg5hS/GWhK8TtSPSRNBeYB04DZwHclVZ9Z883MrCfKSvqSxgN/CtydxgW8D3gwVVkK3JCG56ZxUvm1qf5cYFlEHIuIF4B6YEYvLIOZmZWp3CP9bwN/C7Sm8dHAwYhoTuMNwLg0PA7YBZDKD6X67fFOpmknaYGkvKR8Y2Nj+UtiZmbd6jbpS/oAsC8i6vqhPUTEkojIRUSutrbTK47MzOw0lXPJ5nuAP5M0BzgHeAvwT8BISUPS0fx4YHeqvxuYADRIGgJcCOwvircpnqbX1e1sYs32/cycPJorL63pq9mYmZ1Vuj3Sj4hFETE+IiZSOBH7RETcCKwGPpSqzQceScMr0jip/IkoPL95BTAvXd0zCZgCPNVrS1KkbmcTN969hm/+fCs33r2Gup1NfTEbM7Ozzplcp/954LOS6in02d+T4vcAo1P8s8BCgIjYBCwHNgP/BtwSES1nMP8urdm+n+PNrbQGnGhuZc32/X0xGzOzs06P7siNiF8Cv0zD2+nk6puIeAP4yy6m/zrw9Z42sqdmTh7NsCFVnGhuZeiQKmZOHt3XszQzOysM6scwnK4rL63hvk/MdJ++mVkHFZn0oZD4nezNzE7mZ++YmWWIk76ZWYY46ZuZZYiTvplZhjjpm5lliJO+mVmGOOmbmWWIk76ZWYY46ZuZZYiTvplZhjjpm5lliJO+mVmGOOmbmWWIk76ZWYaU82L0cyQ9JelZSZskfTXFvy/pBUnr02d6ikvSnZLqJT0n6Yqi75ovaVv6zO9ilmZm1kfKeZ7+MeB9EfGqpKHAf0j6WSr7nxHxYIf611N4/+0U4CrgLuAqSaOALwM5IIA6SSsiwi+wNTPrJ+W8GD0i4tU0OjR9osQkc4EfpOnWACMljQWuA1ZFxIGU6FcBs8+s+WZm1hNl9elLqpa0HthHIXGvTUVfT104d0ganmLjgF1FkzekWFfxjvNaICkvKd/Y2NizpTEzs5LKSvoR0RIR04HxwAxJ7wIWAb8LvBsYBXy+NxoUEUsiIhcRudra2t74SjMzS3p09U5EHARWA7MjYk/qwjkG/AswI1XbDUwommx8inUVNzOzflLO1Tu1kkam4XOB9wO/Sf30SBJwA7AxTbICuCldxTMTOBQRe4DHgFmSaiTVALNSzMzM+kk5V++MBZZKqqawk1geEY9KekJSLSBgPfCpVH8lMAeoB44CHweIiAOSvgasS/Vui4gDvbYkZmbWLUWUuhBnYOVyucjn8wPdDDOzs4qkuojIdVbmO3LNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ8p5XeI5kp6S9KykTZK+muKTJK2VVC/pAUnDUnx4Gq9P5ROLvmtRim+VdF2fLZWZmXWqnCP9Y8D7IuL3genA7PTu228Ad0TE24Em4OZU/2agKcXvSPWQNBWYB0wDZgPfTa9gNDOzftJt0o+CV9Po0PQJ4H3Agym+lMLL0QHmpnFS+bXp5elzgWURcSwiXqDwDt0ZvbEQZmZWnrL69CVVS1oP7ANWAc8DByOiOVVpAMal4XHALoBUfggYXRzvZJrieS2QlJeUb2xs7PECmZlZ18pK+hHREhHTgfEUjs5/t68aFBFLIiIXEbna2tq+mo2ZWSb16OqdiDgIrAauBkZKGpKKxgO70/BuYAJAKr8Q2F8c72QaMzPrB+VcvVMraWQaPhd4P7CFQvL/UKo2H3gkDa9I46TyJyIiUnxeurpnEjAFeKqXlsPMzMowpPsqjAWWpittqoDlEfGopM3AMkl/DzwD3JPq3wP8UFI9cIDCFTtExCZJy4HNQDNwS0S09O7imJlZKSochA9OuVwu8vn8QDfDzOysIqkuInKdlfmOXDOzDHHSNzPLECd9M7MMcdI3M8sQJ30zswxx0jczyxAnfTOzDHHSNzPLECd9M7MMcdI3M8sQJ30zswxx0jczyxAnfTOzDHHSNzPLECd9M7MMcdI3M8uQcl6XOEHSakmbJW2S9JkU/4qk3ZLWp8+comkWSaqXtFXSdUXx2SlWL2lh3yySmZl1pZzXJTYDn4uIpyVdANRJWpXK7oiIfyyuLGkqhVckTgPeCvxC0jtS8XcovGO3AVgnaUVEbO6NBTEzs+51m/QjYg+wJw0fkbQFGFdikrnAsog4BryQ3pU7I5XVR8R2AEnLUl0nfTOzftKjPn1JE4HLgbUp9GlJz0m6V1JNio0DdhVN1pBiXcXNzKyflJ30JZ0PPATcGhGHgbuAtwHTKfwS+GZvNEjSAkl5SfnGxsbe+EozM0vKSvqShlJI+PdFxI8BImJvRLRERCvwz7zZhbMbmFA0+fgU6yp+kohYEhG5iMjV1tb2dHnMzKyEcq7eEXAPsCUivlUUH1tU7c+BjWl4BTBP0nBJk4ApwFPAOmCKpEmShlE42buidxbDzMzKUc7VO+8B/grYIGl9in0B+Kik6UAAO4BPAkTEJknLKZygbQZuiYgWAEmfBh4DqoF7I2JTry2JmZl1SxEx0G3oUi6Xi3w+P9DNMDM7q0iqi4hcZ2W+I9fMLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDynlH7gRJqyVtlrRJ0mdSfJSkVZK2pb81KS5Jd0qql/ScpCuKvmt+qr9N0vy+WywzM+tMOUf6zcDnImIqMBO4RdJUYCHweERMAR5P4wDXU3gZ+hRgAXAXFHYSwJeBq4AZwJfbdhRmZtY/uk36EbEnIp5Ow0eALcA4YC6wNFVbCtyQhucCP4iCNcBISWOB64BVEXEgIpqAVcDs3lwYMzMrrUd9+pImApcDa4ExEbEnFb0MjEnD44BdRZM1pFhX8Y7zWCApLynf2NjYk+aZmVk3yk76ks4HHgJujYjDxWUREUD0RoMiYklE5CIiV1tb2xtfaWZmSVlJX9JQCgn/voj4cQrvTd02pL/7Unw3MKFo8vEp1lXczMz6STlX7wi4B9gSEd8qKloBtF2BMx94pCh+U7qKZyZwKHUDPQbMklSTTuDOSjEzM+snQ8qo8x7gr4ANktan2BeA24Hlkm4GdgIfTmUrgTlAPXAU+DhARByQ9DVgXap3W0Qc6I2FMDOz8qjQHT845XK5yOfzA90MM7OziqS6iMh1VuY7cs3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDynld4r2S9knaWBT7iqTdktanz5yiskWS6iVtlXRdUXx2itVLWtj7i2JmZt0p50j/+8DsTuJ3RMT09FkJIGkqMA+Ylqb5rqRqSdXAd4DrganAR1NdMzPrR92+Izcifi1pYpnfNxdYFhHHgBck1QMzUll9RGwHkLQs1d3c8yabmdnpOpM+/U9Lei51/9Sk2DhgV1GdhhTrKn4KSQsk5SXlGxsbz6B5ZmbW0ekm/buAtwHTgT3AN3urQRGxJCJyEZGrra3tra81MzPK6N7pTETsbRuW9M/Ao2l0NzChqOr4FKNE3MzM+slpHelLGls0+udA25U9K4B5koZLmgRMAZ4C1gFTJE2SNIzCyd4Vp99sMzM7Hd0e6Uv6EXANcJGkBuDLwDWSpgMB7AA+CRARmyQtp3CCthm4JSJa0vd8GngMqAbujYhNvb0wZmZWmiJioNvQpVwuF/l8fqCbYWZ2VpFUFxG5zsp8R66ZWYY46ZuZZYiTvplZhjjpm5lliJO+mVmGOOmbmWWIk76ZWYY46ZuZZYiTvplZhjjpm5lliJO+mVmGOOmbmWWIk76ZWYY46ZuZZYiTvplZhjjpm5llSLdJX9K9kvZJ2lgUGyVplaRt6W9NikvSnZLqJT0n6Yqiaean+tskze+bxTEzs1LKOdL/PjC7Q2wh8HhETAEeT+MA11N4L+4UYAFwFxR2EhRes3gVMAP4ctuOwszM+k+3ST8ifg0c6BCeCyxNw0uBG4riP4iCNcDI9BL164BVEXEgIpqAVZy6IzEzsz52un36YyJiTxp+GRiThscBu4rqNaRYV/FTSFogKS8p39jYeJrNMzOzzpzxidwovFm9196uHhFLIiIXEbna2tre+lozM+P0k/7e1G1D+rsvxXcDE4rqjU+xruJmZtaPTjfprwDarsCZDzxSFL8pXcUzEziUuoEeA2ZJqkkncGelmJmZ9aMh3VWQ9CPgGuAiSQ0UrsK5HVgu6WZgJ/DhVH0lMAeoB44CHweIiAOSvgasS/Vui4iOJ4fNzKyPqdAlPzjlcrnI5/OnNW3dzibWbN/PzMmjufJSXx1qZtkhqS4icp2VdXukfzaq29nEjXev4XhzK8OGVHHfJ2Y68ZuZUaGPYVizfT/Hm1tpDTjR3Mqa7fsHuklmZoNCRSb9mZNHM2xIFdWCoUOqmDl59EA3ycxsUKjI7p0rL63hvk/MdJ++mVkHFZn0oZD4nezNzE5Wkd07ZmbWuYpN+nU7m/jO6nrqdjYNdFPMzAaNiuze8SWbZmadq8gj/eJLNo+daOXbv/itj/jNzKjQI/2aEcNoTTcaB/Af215h7fb9/GVuAh+8YnynR/2+g9fMsqAik/7Glw6dNB7A8Zbg/rUv8tDTDad097g7yMyyoiK7d9RFPOj8Dl3fwWtmWVGRSX/aWy88aXzGxJqSd+j6Dl4zy4qK7N5pOnq8fVjAH112MZ+//p1d9tn7Dl4zy4qKTPo1I4a1D0eH8a74Dl4zy4KKTPodT+Su3rqP2x7d5BO1ZpZ5Z9SnL2mHpA2S1kvKp9goSaskbUt/a1Jcku6UVC/pOUlX9MYCdNquDuMbGg76RK2ZGb1zIvePI2J60VtaFgKPR8QU4PE0DnA9MCV9FgB39cK8O/XBK8YzbMibi7b38DFaA6p8otbMMq4vrt6ZCyxNw0uBG4riP4iCNcBISWP7YP5ceWkNX/nP05g4egSi0K9fBbzn7Re1d+342TxmlkVn2qcfwM8lBbA4IpYAYyJiTyp/GRiThscBu4qmbUixPfSyup1N3PboJo6daC0kfMGwIVXc+ifvaE/4vhnLzLLoTJP+eyNit6SLgVWSflNcGBGRdghlk7SAQvcPl1xyyWk1as32/bxxorV9fHLt+XzjL/5Te2Lv7GYsJ30zy4Iz6t6JiN3p7z7gJ8AMYG9bt036uy9V3w1MKJp8fIp1/M4lEZGLiFxtbe1ptWvb3iMnjdfve5XFv3q+vSvHN2OZWVaddtKXdJ6kC9qGgVnARmAFMD9Vmw88koZXADelq3hmAoeKuoF61fpdB0+J/XzzXm68ew11O5vab8b67KzL3LXTD3z+xGzwOJPunTHATyS1fc/9EfFvktYByyXdDOwEPpzqrwTmAPXAUeDjZzDvki4ZNYId+4+eEn/jRCu33FfHf7/2HXzsqkuc7PuBz5+YDS6nnfQjYjvw+53E9wPXdhIP4JbTnV9PHHr9RJdlLx8+xhd+soEX97/GBecO9WMX+pjPn5gNLhV5R+7FbzkHOFSyzuJfbwegukrcNvddXPY7F/jZO32g7fzJieZWnz8xGwQqMun/8WUXs2rz3pJ12i4pam4N/tfDG6iurqK5xV0Qvc0PszMbXCoy6TcdPd5+U1Y5WgNam1tPet6+k1Pv8cPsyuc3uFlfq8ik39MuhKoqABER7oLoA05k5fFJb+sPFZn0f/jkjrKP8gFaWgGCKsFfXz3R/9G60ZMk7kRWPp/0tv5QkW/O+umG0pf/VwnefvH5p8RbAxb/+/ayryevhOvPb1+5hWv+YTW3r9xSVv26nU3MW/Ik//DYVuYtebLbZferKMvnmwatP1Tkkf6JltLH+a0Bz+97tdOyCPjGz7aw/FN/UPI76nY28ZElT9LcEgypFg8suPqsOyq7feUWvpeuYmr7u3DOO0tOs/hXz7ev3xMtwecfeu6kR1x0VJy4gp53vWWJT3pbf6jII/1ylNot5Ms4cl/8q+dpTsmvuSVY/Kvne6ll/ef+p14sOd6ZtTtOPlKv3/cqH1n8JPevfXPatl9A9699kb/7yQZa08pujULXWyX8QuotXhfW3yrySP9MtXayR+jYj/3UjgMnle89/EY/te70dVyGo8ebTypvG79/7Yv8bOMern/X2FPuXzjyevMp39vcGnwx3fB2yejz+NIjG2lpjU53rA+vf4mVG1/25bGcer7jSx+Yxlf+dVP7PQ0/+q8zAdrXf/FwVteZnTkn/S68bdFPqb1gODMnj2b3wddZt6NwJFYtmHzReRw8evJdv/tfPdY+XLezie/96nn2HX6Dj7z7Ej521ek9LfR03LrsGX7520aueUct3553+Ult6nhCtbn15GmbW+Gme9by622vAPDv6W+bC4ZXd7pDhMIvp7Yuou4cTzM+XtTHX5zMBnL99bXiHe+Pn25of/z3ieZWHlj34knr5vMPPsuLTa/T3NLKkCqB5J2lnTEn/S60ROGRDQ+vf+mU+LbG106p33DwDaZ/9THOO2coew6+3p4cn23YAMDHrrqk5FUvHcvaEt8Lja8yufZ8PvlHb+v2P/mty55pb2/b32/Pu7zwfoF/3dT+uOm2ZxB15tcdEn2xI8daSs6/p1qj8ETUO37xW5pbguoquPKSGtbtaGr/lfBsQ9ePzKjb2cRDTzcgYNpbL6Tp6PFePwou/neB7o+0u6oP8NDTDSzP76IlLStS+3JWV1e1J/w29UXb2fGWoK1T8tiJVh56uqF9O/HRv/WECo/EGZxyuVzk8/keTzdx4U/7oDWnb2i1GD1iGC8fefPXwKf+cDIL57yzPXE9WNfQfhT3pQ9M4+8e3nDSUXVV+oUxufZ8rrnsYja+dIgnNu/lldeOMb5mBN/88HTmLXnylJPYPblJ7Wwg4Lxh1ahKHHnj1K4mgCm157Hqc9e0j7clxpoRw1i9dV/JXxAdk/Zf3PX/2suqVNhRVQu+dsPvnbSTuXXZM6zcsIfjLYFS3eJ/iu7+HS4YXt3jneqlo0aw88CbDxacNXUMn/yjtwG07ww/eMX4bg8wKkV3y9VWfuT1E2zac5jr3zW2on5FFpNUV/QK25PLnPStUl06agT7jrzBG6kLpaNRI4YyYdQIPvLuS3hx/2s8vH43ew8f6/FOcli10pH44DRjYiEBbn/lNY4eb+GNEy1Eemf0te98c0fRtmMs9YupnB1GV3WKd77lzKNmxDA2vnSI+r1HONbcyqSLzuOFV17jeHMrLx44yrHmVt711rfw/mm/w7a9R1jx7EvtO+V3jLmAlw69zmvHWhh13lA+ePl4vv/kjpNergRQXQWXjbmAr93we+3roPjXdsedZ7k7lp7uUHt7R+ykb2Y9NmrEUA6/cYIqiWqJE61Bc/r5ObRaXD15NOt2HGjfqY48dwgBHCo62f+pP5zMk9v3s+vAUQ4cPfXptzMm1rD74Ovsf+34KQl5sOm4cz93aBXDhlZxojmogpN+fQp4d1o2JKaNfUt7F23bzqR+7xEOvHacodVV/Hbfq7S2BtVV4hPvncThY81d/lIrh5O+mdlZ6qG/+YMeJ/5SST+z1+mbmZ0NPrbkyV79Pid9M7NB7Fgvny/q96QvabakrZLqJS3s7/mbmWVZvyZ9SdXAd4DrganARyVN7c82mJllWX8f6c8A6iNie0QcB5YBc/u5DWZmmdXfSX8csKtovCHF2klaICkvKd/Y2NivjTMzq3SD7kRuRCyJiFxE5Gprawe6OWZmFaW/k/5uYELR+PgU61U7bv/T3v5KM7MB0dv5rL8fuLYOmCJpEoVkPw/4WF/MyInfzOxU/Zr0I6JZ0qeBx4Bq4N6I2NSfbTAzy7J+f7RyRKwEVvb3fM3MbBCeyDUzs77jpG9mliFO+mZmGeKkb2aWIYP6efqSGoGdZ/AVFwFdv/TVvH5K8/opzeuntIFcP5dGRKd3tw7qpH+mJOW7epGAef10x+unNK+f0gbr+nH3jplZhjjpm5llSKUn/SUD3YBBzuunNK+f0rx+ShuU66ei+/TNzOxklX6kb2ZmRZz0zcwypCKTfqW/fF3SBEmrJW2WtEnSZ1J8lKRVkralvzUpLkl3pvXxnKQrir5rfqq/TdL8oviVkjakae6UpFLzGGwkVUt6RtKjaXySpLVpeR6QNCzFh6fx+lQ+seg7FqX4VknXFcU73b66msdgJGmkpAcl/UbSFklXe/t5k6T/kf5vbZT0I0nnVMw2FBEV9aHwyObngcnAMOBZYOpAt6uXl3EscEUavgD4LYUXzf9vYGGKLwS+kYbnAD8DBMwE1qb4KGB7+luThmtS2VOprtK016d4p/MYbB/gs8D9wKNpfDkwLw1/D/ibNPzfgO+l4XnAA2l4atp2hgOT0jZVXWr76moeg/EDLAU+kYaHASO9/bSvm3HAC8C5Rf+uf10p29CAr+A++Ae7GnisaHwRsGig29XHy/wI8H5gKzA2xcYCW9PwYuCjRfW3pvKPAouL4otTbCzwm6J4e72u5jGYPhTeyPY48D7g0ZR4XgGGdNxGKLzb4eo0PCTVU8ftpq1eV9tXqXkMtg9wYUpq6hD39hPtSX8XhZ3ZkLQNXVcp21Aldu90+/L1SpJ+Sl4OrAXGRMSeVPQyMCYNd7VOSsUbOolTYh6DybeBvwVa0/ho4GBENKfx4uVpXwep/FCq39N1Vmoeg80koBH4l9QFdrek8/D2A0BE7Ab+EXgR2ENhm6ijQrahSkz6mSHpfOAh4NaIOFxcFoVDhT69Hrc/5tFTkj4A7IuIuoFuyyA2BLgCuCsiLgdeo9DV0i6r2w9AOs8wl8LO8a3AecDsAW1UL6rEpN8vL18faJKGUkj490XEj1N4r6SxqXwssC/Fu1onpeLjO4mXmsdg8R7gzyTtAJZR6OL5J2CkpLY3xRUvT/s6SOUXAvvp+TrbX2Ieg00D0BARa9P4gxR2At5+Cv4EeCEiGiPiBPBjCttVRWxDlZj021++ns58zwNWDHCbelW6EuIeYEtEfKuoaAXQdgXFfAp9/W3xm9JVGDOBQ+kn9mPALEk16ehmFoU+xD3AYUkz07xu6vBdnc1jUIiIRRExPiImUvi3fyIibgRWAx9K1Tqum7bl+VCqHyk+L12ZMQmYQuHkZKfbV5qmq3kMKhHxMrBL0mUpdC2wGW8/bV4EZkoakdrftn4qYxsa6JMmfXQiZg6FK1qeB7440O3pg+V7L4Wfxc8B69NnDoU+wceBbcAvgFGpvoDvpPWxAcgVfdd/AerT5+NF8RywMU3zf3jz7u1O5zEYP8A1vHn1zmQK/+Hqgf8LDE/xc9J4fSqfXDT9F9PybyVdfVJq++pqHoPxA0wH8mkbepjC1Tfeft5s/1eB36Rl+CGFK3AqYhvyYxjMzDKkErt3zMysC076ZmYZ4qRvZpYhTvpmZhnipG9mliFO+mZmGeKkb2aWIf8fimFBHijHfowAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(torch.arange(0,len(flat_fft2)), flat_fft2.abs(), '.')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 62,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "topk = torch.topk(\n",
+ " flat_fft2.abs(), round(0.1*len(flat_fft2)), dim=0, sorted=False\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 63,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "torch.return_types.topk(\n",
+ "values=tensor([ 75.7603, 695.2839, 721.5375, ..., 68.1649, 68.1649, 68.1649]),\n",
+ "indices=tensor([294037, 1, 2, ..., 241565, 434039, 328013]))"
+ ]
+ },
+ "execution_count": 63,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "topk"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 64,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([294037, 1, 2, ..., 241565, 434039, 328013])"
+ ]
+ },
+ "execution_count": 64,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "topk.indices"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 65,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "top10 = torch.zeros(len(flat_fft2), dtype = torch.cfloat)\n",
+ "top10[topk.indices] = flat_fft2[topk.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 66,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "84502"
+ ]
+ },
+ "execution_count": 66,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(topk.indices)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 67,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse_top10 = fft.irfft(top10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 68,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(34.8182)"
+ ]
+ },
+ "execution_count": 68,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_top10 - conc2, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 69,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 69,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc2, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 70,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(30254.6758)"
+ ]
+ },
+ "execution_count": 70,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_top10 - conc2, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 71,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " conc2.abs(), round(0.1*len(conc2)), dim=0, sorted=False\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 72,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "169005"
+ ]
+ },
+ "execution_count": 72,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(topk_og.indices)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 73,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 1, 2, 3, 4, 0, 6, 7, 8, 9, 10, 11, 12, 13, 16, 18, 19, 20, 21,\n",
+ " 22, 23, 24, 25, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 38, 40, 41, 42,\n",
+ " 44, 45, 46, 47, 48, 49, 39])"
+ ]
+ },
+ "execution_count": 73,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "topk_og.indices[topk_og.indices<50]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 74,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "top10_og = torch.zeros(len(conc2))\n",
+ "top10_og[topk_og.indices] = conc2[topk_og.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 75,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(15.5541)"
+ ]
+ },
+ "execution_count": 75,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(top10_og - conc2, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 76,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(15858.2695)"
+ ]
+ },
+ "execution_count": 76,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(top10_og - conc2, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 77,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0058, -0.0441, -0.0381, -0.0493, -0.0240, 0.0190, -0.0132, -0.0221,\n",
+ " -0.0472, 0.0077, 0.0236, 0.0183, 0.0231, 0.0014, -0.0245, -0.0085,\n",
+ " 0.0035, -0.0036, 0.0150, 0.0107, 0.0123, 0.0039, 0.0003, -0.0320,\n",
+ " -0.0093, 0.0632, 0.0360, 0.0200, -0.0248, 0.0029, -0.0011, -0.0193,\n",
+ " 0.0221, 0.0056, -0.0091, -0.0008, 0.0329, 0.0133, -0.0078, -0.0061,\n",
+ " -0.0372, -0.0354, -0.0238, -0.0028, 0.0145, -0.0121, -0.0517, -0.0468,\n",
+ " -0.0123, -0.0132])"
+ ]
+ },
+ "execution_count": 77,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "reverse_top10[10000:10050]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 78,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0155, 0.0128, -0.0386, 0.0186, 0.0166, 0.0259, -0.0200, -0.0033,\n",
+ " -0.0399, 0.0214, 0.0106, 0.0197, -0.0182, -0.0191, -0.0370, 0.0159,\n",
+ " 0.0071, -0.0321, -0.0166, -0.0082, 0.0090, 0.0291, 0.0117, 0.0011,\n",
+ " 0.0066, 0.0163, 0.0237, 0.0092, -0.0029, -0.0209, -0.0207, 0.0039,\n",
+ " 0.0065, 0.0057, 0.0316, -0.0262, -0.0342, -0.0115, 0.0149, -0.0175,\n",
+ " -0.0568, -0.0135, -0.0503, -0.0252, 0.0148, -0.0429, -0.0424, -0.0182,\n",
+ " -0.0002, -0.0341])"
+ ]
+ },
+ "execution_count": 78,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "conc2 [10000:10050]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 79,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0000, 0.0000, -0.0386, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " -0.0399, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, -0.0370, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " -0.0568, 0.0000, -0.0503, 0.0000, 0.0000, -0.0429, -0.0424, 0.0000,\n",
+ " 0.0000, 0.0000])"
+ ]
+ },
+ "execution_count": 79,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "top10_og[10000:10050]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 80,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1440, -0.0482, 0.2070, ..., 0.0011, 0.0177, -0.0218])"
+ ]
+ },
+ "execution_count": 80,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "conc2[0:10000]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 81,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse_top10fft = reverse_top10"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 82,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACSL0lEQVR4nO2dd3hjZ5m371fdsizJ3dM9vU8myaQnMKQTQghtISSQLCVLZ1nY3SzLtwQIEFggIRD6QkIvgRRqSO+ZZJLMZPp4+njGvUiWZPX3++M9R82SLNmyPWOf+7p82To60nklS+c5T/s9QkqJgYGBgcHMxTTVCzAwMDAwmFoMQ2BgYGAwwzEMgYGBgcEMxzAEBgYGBjMcwxAYGBgYzHAMQ2BgYGAwwzEMgYHBSY4QYqMQon2q12Fw8mIYAoMpRQhxSAgxLIQICCG6hBB3CSFcU72uYgghWoUQUghhqdDzOYQQg0KIC/Pcd5sQ4p5KHMfAoBCGITA4EXijlNIFnAZsAD5bzoOF4qT5LOcaECllGPgt8J6c/czANcDdk7c6g5nISfPlMZj+SCmPAX8D1gghaoUQfxZC9AghBrS/5+r7CiEeF0J8SQjxDBACFgkh/lkIsUsIMSSEOCCE+JeM/TcKIdqFEP8hhOgWQnQIIa4WQlwhhNgrhOgXQnwmY3+TEOImIcR+IUSfEOJ3Qog67e4ntd+DmidzjvaY92rHHxBCPCiEWJDxfFII8REhRBvQlufl3w28VQjhzNh2Geo7+rdiry0X7VhLMm7fJYS4JeP2lUKILZoX8qwQYl3Gff8phDimHWePEOKiQscxmD4YhsDghEEIMQ+4AngF9dn8KbAAmA8MA9/Jeci7gRuBGuAw0A1cCbiBfwZuE0KclrF/C+AA5gD/A/wIuA44HbgA+H9CiIXavh8DrgZeC8wGBoA7tfteo/32SildUsrnhBBvAj4DvAVoBJ4Cfp2z3quBs4BVua9dSvks0KE9PvP1/UpKGS/htZWEEOJU4CfAvwD1wA+AB4QQdiHEcuCjwBlSyhqUITpU7jEMTkKklMaP8TNlP6gTTQAYRJ3MvwtU5dlvPTCQcftx4AujPPd9wCe0vzeijIlZu10DSOCsjP1fAq7W/t4FXJRx3ywgBliAVu2xloz7/wa8L+O2CeWpLNBuS+DCUdb7WeAf2t9u7fGnlvja2jPuk8CSjNt3Abdof38P+GLOc+1BGbwlKINzMWCd6s+G8TN5P4ZHYHAicLWU0iulXCCl/LCUclgI4RRC/EAIcVgI4UeFY7xa3FznaOaTCCFeL4R4XgvzDKK8i4aMXfqklAnt72Htd1fG/cOAnqheANyrhU8GUYYhATQXeA0LgG9l7N8PCJT3kXe9efg58DohxGzgbcB+KeUrJb62UlkAfEpfp/Zc84DZUsp9wL8CNwPdQojfaGsxmOYYhsDgROVTwHLUFbubdDhGZOyTks4VQtiBPwBfB5qllF7grzn7l8NR4PWagdJ/HFLlMfJJ9h4F/iVn/yqpQj4j1psPKeVhVEjpOlRY6O4xvrYQkJlraMlZ55dy1umUUv5aW8OvpJTnowyGBL5abM0G0wPDEBicqNSgrtAHtSTt50bZ3wbYgR4gLoR4PXDpOI7/feBLesJXCNGo5QHQjpEEFuXs/19CiNXa/h4hxNvHcNy7UXH684BfatvKfW1bgHcJIcxCiMtRYR+dHwEfFEKcpVVbVQsh3iCEqBFCLBdCXKgZnjDq/U+O4TUYnGQYhsDgROV2oAroBZ4H/l5sZynlEPBx4HeoxO67gAfGcfxvaY//hxBiSFvDWdqxQsCXgGe08MrZUsp7UVfPv9FCWduB14/huH8A6oBHpJQdY3xtnwDeiMq7XIvKJ6A912bgA6jE+wCwD7hBu9sO3Ip6zzuBJuC/xvAaDE4yhJTGYBoDAwODmYzhERgYGBjMcAxDYGBgYDDDMQyBgYGBwQzHMAQGBgYGM5yKqCdONg0NDbK1tXWql2FgYGBwUvHSSy/1Sikbc7eflIagtbWVzZs3T/UyDAwMDE4qhBCH8203QkMGBgYGMxzDEBgYGBjMcAxDYGBgYDDDMQyBgYGBwQzHMAQGBgYGMxzDEBgYGBjMcAxDYGBgYDDDqYghEEJcrg263ieEuCnP/XYhxG+1+zcJIVq17ZcIIV4SQmzTfl9YifUYGBhMMD174OCTU70KgwoxbkOgjQ68E6W9vgq4RgiRO5z7fah5s0uA20hPPeoF3iilXAtcjxrVZ2BgcKLz+K3wwMemehUGFaISHsGZwD4p5QEpZRT4DfCmnH3ehDZ2D7gHuEgIIaSUr0gpj2vbdwBV2nQkAwODE5lQH0SGpnoVBhWiEoZgDtlDudvJHtidtY+UMg74gPqcfd4KvCyljOQ7iBDiRiHEZiHE5p6engos28DAYMyEByEamupVGFSIEyJZrM15/SrwL4X2kVL+UEq5QUq5obFxhGaSgYHBZDI8APFhSCameiUGFaAShuAYMC/j9lxtW959hBAWwAP0abfnAvcC75FS7q/AeqY30SD87Gro3j3VKzGYyQwPqt8xwyuYDlTCELwILBVCLBRC2IB3MnKw9gOoZDDA24BHpZRSCOEF/gLcJKV8pgJrmf70tsGBx+DopqleicFMJRGHiF/9bYSHJo1n9vXy5b/uYjhaeS9s3IZAi/l/FHgQ2AX8Tkq5QwjxBSHEVdpu/wfUCyH2Af8G6CWmHwWWAP8jhNii/TSNd03TmmCv+h0NTO06DGYuYV/6b+NzOGm8cLCfHz11AKtZVPy5KzKPQEr5V+CvOdv+J+PvMPD2PI+7BbilEmuYMYQ0QxAxvoAGU0R4MP23ERqaNHoCEeqrbVjMlU/tnhDJYoMySHkERumewRQxPJD+OxqcunXMMHqGIjS4Jqa63jAEJxu6R2B8AQ2mCsMQTAk9QxEaawxDYAAQ1HoojNCQwVShVwyBYQgmkZ6hCI2GR2AAQLBP/TaSdAZTRaZHYOQIJgUpJb2BifMITsrh9TOaVLLYyBHoPLyzi5eODFBtM+O0Wai2p38vbaphXp1zqpc4vcgKDRkXJJPBUCROJJ40DIGBhh4aMr6AKT73wA6ODQ6P2H6OaQdnVHXyyc9+AyEqX3I3YwkPgjCDTBh9BJNEz5BS3pmoZLFhCE42UqEhIzYLEIjEOTY4zKcvXcaNr1lMKBonFE0Qisax/eLrNPq20+X7Ei3eqqle6vRheABqWsB/zPgcThK6ITCSxQYQj6TLRo1kMQD7utX7sLS5BpvFhNdpY7a3iiVeE3MD26gSUdoOtE3xKqcZw4PgrANLFcQMQzAZGIbAII3eQ2B1GqEhjb1dyjAubXJl33H4WUzJGADdhwxdpooyPABVtWBzGh7BJJEyBEbVkEEqP1DbqgyBlFO6nBOBfd0BbBYTC+qrs+/Y/1jqz0DH3kle1TRneAAcXrBVGzmCSaI3EMFqFniqrBPy/IYhOJnQK4ZqW0EmjdI9lEewuNGF2ZSTDD7wOCw4nwQmGDg4JWs7mYnEEzy2u5v/uX87T+zNmf8RHlQegbXa8EwniZ6hCPXVdky5n/MKYSSLTyb0RHFtq/odDaqrshlMW1eA0xfUZm8c6oLuHXDxzQS6DlIbPIZvODZhV1PTBX84xmO7u/nHzi4e391NUFO57PCFee0ybQaIlBmhoWrjYmSS6JnAHgIwDMHJRWZoCFQvgasEsdaBw5CMQ/3iCVvaVBDUKoauOXNe9h0HHle/F72OxPaHmB/qYFeHn7MX5Q7FMwBIJCUf/dXLPLyri1hC0uCycdX62Vy6qoXvP7Gf3kDG0MBYCBJRqPIaOYJJpGcoQrPbMWHPbxiCk4lQL5gs4J6tbpfqlv/tPyHYDR94dOLWNgXoFUNLmmqy7zjwGDjroWUdjuYltHZs4Q/HZ44h2N8T4LHd3bz/gkUl7X+wN8Dftndy5bpZ3HBuK6fOr02F2u7fcoyXjwymd9blJapqweaC0MCI5zOoPD1DEdbM9kzY8xs5gpOJYC84G8CunfhKLSENdquBNtMsuaxXDC1rzqgYklIlihe+FkwmnM1L8YogB462T9EqJ58/vtzOLX/Zlao0GY09nepz9MHXLmZDa11WvqXeZc/2CPSu4qpao3ptkkgmJX3B6ISGhgxDcDIR6oPqBrBphqDUL2HYryZKDU+vq7e27gA2s4n5mRISPXsg0AmLNqrbtQsB8B+bOZVDgyFVNrunszQZkj1dQ5gELMktwUV1suoNekB6FoHDq0JDMy1HEOyDoy9M6iEHQlESSRWymygMQ3AyEexRIQ+79oUt1RDoYwUHD0/MuqaItq4hFjVWZw/qOKCVjS5+nfpdpwyBGDxEODYzBq3rhmB3p7+k/fd2DtFaX43Dah5xX7128ukLRNWGTI/A5pp55aPP3gE/ff2kXlT1BPRmsonLERiG4GQi2AvVjelKoVJDQ7pA3cChCVnWVLG3K8Cy5pz8wP7HoG4xeOer21pifZ7spK1rZoQxBofVSXt3iR7B3u4hljaP9AYg3cCkn4zShsCbDg1Ns5BjUXxHVeFFRp/KRDPRXcVgGIKTi1RoqAyPIBFLu+/TyBDoFUNZHcXxKBx6Ou0NANiqiTubWSC62HHcN/KJpiEDwdJDQ+FYgkO9QZbnGlQNXeQs7REMqt96+SgS4uHxLvnkwd+hfu97eNIOaRgCgzTxiArxODMMQSkeQaZc9TQyBJkaQynaX1TaN3p+QMNcv4iF5m52HC8tVHKy4xtWhmBv1xCJZPGr9f09AZISlrXkNwR6aKg30yMwWdRnUPdMZ1IJ6VCGIUgmJ+WQvQHDEBjo6DpD1fVgtijBr1LmFoczroIHpk+OIKUxlBnSOPAYCBO0XpC1r6hbyCJzDzs7ZoYhGAhF8TqtROJJDvcVP0nr72MhjyCdI9AMQXhQJYqFqJgheP5AH4Oh6LieY1KQUhkCVzMEuqBr26QctmcogsNqoto2ModTKQxDcLKgy0tUax2edldpX0A9UWypmnYegc1sYkFmxdCBx2HO6Sp+nUndQuqTvRzo6B31CnmiiSeSbDrQN2HPH4knCEUTnLWwDhg9PLS3K4DVLGhtyN+hbreYqXFY6M1MFldpndxW7b0fhyEIxxJc9+NN/OipA2N+jkkjPKjCYKdco263/WNijvPIF+C316Vu6rOKJ3KmhmEIThZ0j8DZoH7bqssLDTWv1hJd06NyZm9uxdDwIBx7CRa9buTOdaqxqiHWwaFRrpAnmj+/2sE7fvg8Lx3un5Dn92kVQ2e01mESoyeM93YOsajBhdVc+FTQmNlLkGkI9BDlOEpIewMR4knJ7o6TYOKenh+YtQ5mrYe2CcoTHH0Bdv8llZjvCUzcrGIdwxCcLKRCQ7ohqCktWRzWPIJZ61S1g//YxKxvkmnrDmTnBw49pYT4cvIDQKqXQCWMpzY8tOXoIAAP7+qekOcf0AxBi8dBa331qB7Bnq6hgvkBnXqXLcMQDKY9LpvuEYy9GktPhLZ1nwQVXXp+oGYWLL0E2l+YmDLSUL/6LB98Ekh7BBOJYQhOFkI5hsDuKm1usR4aalmrfk+D8FAwEqd9IKdiaP9jSg1z7hkjH6D1Eiw0d7Nzig2BfvzHdk+MIdBj7bVOG8tbakb2EsTSFT76+7i8QOmoToPLnt1HkPII9BzB2D0C3RAcHQgxHD3BvdWhTvW7ZhYsvVSdrPdPgGyL/l3XntswBAZpgprOkMOrbttc5XkELaeo39PAEOzvUa97WW6iuPV8sOTpvqyqBYeHU5z9U1pCmkxKdhz34bSZ2d05lHfO8njRPQJPlZXlLTUc7g+lu4IPPwtfmQN9+4H0VfjSAolinYas0NBg+jNoHX+yWM89SJn+v56wDB1Xv2tatFxUbeXDQ1KqMnGAfY8SiycYCMUmbFaxjmEIThb0rmI9YWR3lZgj0AxB0wo1cHwaVA7t7co5gQ0chv4D2f0DmQgBtQtZau1h53E/cooaoA73hwhGE1x39gIAHp0Ar8CnNZPVVttY0VKDlKQb6V75pQoPavMZ9nYWrxjSqXfZGAjFiMViEPGN9AjGMa4yUw9Jr2A6YRnq1DSWqsBkhsUXVr6MNOJX/6O6ReA7wmD7HmBiS0fBMAQnD6G+dKIYtOlQJVYNme1qf8/caeERtHUNZVcMpWSnNxZ+UN1CZiU66AtG6fKXJsZWaXRv5KpTZrOg3jkh4SHdI/BWWVne4ga0yqF4BHb9Se2klRTv6RrCYTUxL7PyKg/61ehgvyaDnjIE468a6gmEqXFYsJrFiZ8n8HeosJDO0kuVoGPn1sodQ/cG1r4dgOjeR4CJG1GpYxiCk4Vgbzo/AOUlix3qhEBt6/QwBN2B7IqhA4+pL2jjisIPql2IK9yBmQQ7O6YmPLT9mB+rWbC02cXrljfxzL7eisfFB0MxbGYTTpuZ+XVOHFaTqhza/6i6moeUIdjbNcTSppqR091y0MXO0obAq35bK5MjmOVxsLChmrZcj2D7H+DAE2N+7oozlGMIFl+kflcyPBTSqsnmbADvAmyHlZSF4REYKEI5hsDuKk3nJeIH+wloCNoehh9dqCQwymRv11C2Umb7Zph/Tjpslo+6RZhknNmilx3HpiZhvOO4j6VNNdgtZi5c0UQknuS5A70VPcZgKIrHaUUIgdkkWNZcw54uvzqp6p8DLW+0t6uwxlAmukcQGMjxCCw2MFnHXTXUWGNnaVNNtkew7R64573wzO1jfu6KM9RBp6zlv/64TYUXXY0w+1TY91DljpEqE6+HxRfi7XoeC3HDEBho6LMIdGyu0uYWR4bS8wtqFyiDUqpY3UTS9g9V969XYpRIKKoqXVJic5GA6o9oWln8gVrl0Jke35SUkEop2XHcz+rZ6mR81qI6nDZzxfMEg6EYtc70SM7lzTUcPN4Le/4Gq9+sCg7CPgZDKkQ2Wn4A1EwCgJA/xxDAuMdV9gaiNLjsLG12caRfqxw6/Bzc9yG1Q3jqkvtZJBMQ6OKVwSp+/cIR2ge0RP/SS5W0SahCfSF6aMhZB4svxBoPsl7sM5LFBqR1hvSuYkhLUY92Us8NDcGJIUfdq5JgBMs7EaY0hnSPoK9N/W5YVvyBWi/BhhofO6YgNNTpD9MfjLJmjpoyZbeYOX9JA4/u6q5o8nogFMVbla6cWjHLzbrwi+qqfc1blVcQ9qUS7qP1EEA6NBTxayc7vWoISs9V5UFKqTwCl/IIpISjbVvhN9co9dhFrztxDEGgG2SS7X6VF3nhoPZeLLmksmWkuiGoboCFryGJiYvsO/NKhFcSwxCcDKQ+HBmjFktVIM0NDcGJER7q1U7gwfJCI225FUM92sCZxuXFH1gzC8x2Vth7Odo/nBJmmyz0cJTuEQBcuKKJ474weypYLeMbjuHN8AhWtNTwRvOzRB0NqrzW4YGIP2O62+iGwGW3YLeYiAW1k1+uRzBGQxCMJhiOJWissbOs2UUdfmb/5T2quu3a36vP64liCLRmsr0hlRd58ZAeyz8Nquoqp0Ya6gOzTX2/q7wccqzgNaaJ1zQyDMHJgD60Pjc0BKMbgrBfffkhdVU85SWkkaF0h3OgPI9gb/eQ0sap1ypWeveoE0fd4uIPNJmgbiFzUaGoXZMsQLf9uA8hYOWstCF43YomoLJlpAOhKLXODI+gTnChaQttDRepkkeHR/MIhnDZLcz2jD7sRAhBg8tOUg9/ZGo5Wcc+wD5TXnmB28T/2b6BfbgbrvmNKp90eNJ9MFONFsLslHU0uOy8oBsCkxmWXARtD1WmjDTUl1Um/pL5VFYm2yZ8EI5hCMZCPAoP3zx5Vyu58hJQemgoM0dQVauqjabaI9C9ASAx1FXWQ9u6AixqcKUrhnr2qPh/vkayXGoXUhtWs4snO0+w47ifhQ3VVNstqW3Nbgdr5rh5tIJyE4OhbI+gvv1RqkSUxy2aIqtmCPZ0DrGs2VWykFmDy4YI+9QFiDn9/NhcY84RpAyBy4rtgX/hFNM+ftx4E8zTusMdbkhEsrqhpwytmaxX1HLd2fM50BNM90AsuUTl3jq2jP84oX5lCDSeTK7BRBIOPjX+5y6CYQjGwrGX4Onb1FXAZJAKDWXkCEqZW5xMQsTPjj74ydMHtcaq1hPAEKTnBw/2HC/roW2507R690LDKGEhnbqFWHxHaHTZJr3DeMcxH2tme0Zsv3BFMy8fGWAgOH4Z5uFogkg8iTfDI2DHH+k3N/CPIdXEhsON1DyC5SXkB3TqXXYskcHssBCoXoIxVg3p3cqrtn8Ddv2JPzZ8kN8GT03voHuyJ0J4aKiTJCYamudywVL1PdysewVLLgJEZcJDoV6VKNZ4OrSAiMk5MVIWGRiGYCzoJ2Y9ZDPRZJaU6aQ8giLx5WgAkDx3LMp3HtunkpK1C6bcEMievcQx0y4bCPZ3lPy4UDTO0f5hljZpJ7BETHUUN46SKNapWwSxIOe2JCZVc2ggGOW4L5yVH9C5cEUTSQlP7B3/Z0kfUZnyCIYHoe0h9tRfwp7uoJLgdnhIDvsYCMXS72MJNLhs2GL+7EQxaDmCsXsEHgLUv/oDOPU6ji77Z470h9KzpfVjRaY+PCT9x+nBw7r59ayd48FhNaXDQ9UNKldQCVnqjMbR4WiCgQgcqz0D9j8yoSNBK2IIhBCXCyH2CCH2CSFuynO/XQjxW+3+TUKI1oz7/kvbvkcIcVkl1jPh6KJQgfLCGmMm2KPi4LnVGlA8Pqt9gQ4HLfQHoxzqCymPYPDwlM6ZDR7fyaFkM12ylvhQ6WGR/d3qtaY0hvoPqHb8Uj0CLUdypnuQfd2BSRtmr4ehVufxCNbN8dDgslUkT6CPqEyVj+7+CyRjBJdeRTimDalxeFNX2OV6BFUJPzJ31oN17OWjPUMRTjcr3SPWvYOlLTUkMzWHUn0PU+8RhPqO0ZGs5ZR5XmwWE6fOq00njEGFh9o3j7+MVM8RkPaYBmedD4NH1Od9ghi3IRBCmIE7gdcDq4BrhBCrcnZ7HzAgpVwC3AZ8VXvsKuCdwGrgcuC72vOd2OgeQWCSPAK9mcyU8e8qJVmseQv9cZUQfOnwgDIE8fDkGbE8xLv2sE/OIWD2YhkuvWpoxFSyHq0EtWSPQBmCFfY+4kmZrgWfYLZrYah8HoHJJNi4vIkn9vYQT4wv2agrj3r08tHtfwDvAhqXnwNoUhN2N+Z4CAvxkiqGdBpcdjwEiNtyjNk4QkM9QxHOsR9QU+Vmn5paj14ifCKFhuKDx+iWtZw6zwvAGQvr2Hncz1BYqz5bcA4goXMcFT6JuPLiNEPQreUgoq0b1f0HHhv7c4+CGG8NsxDiHOBmKeVl2u3/ApBSfiVjnwe1fZ4TQliATqARuClz38z9ih1zw4YNcvPmzWWv9R1/fgcHfQfLftwIElEVljCZwTJ61cW4iYfVFby1Kr1NSnUlZraqcrN8JBMQDxPBSgITFpMJm0kbNm5xqPVPNlIiYyHiWDAJMMkEWKtKSlpG40niySRVVrPaPxFT/wurs3hXccaxiYVImqyEkybsFvOo8gqVIBJPkExCVYFRg/Fkkmg8Oe716M/jsJrVFZ72+ZAmK8OxBFazCY/JxG8O7uMa+WMe/X9vHvV9//QTn+bJ9ieJJ5OY42EwmREW1dx0/pzz+WbcDc9+B/6nuEHvDHZyzV+uIZghUBeJJbDIKGYBWKuQUqbWaTWbQCa5rq+bj190O6x5y6iv/5XuV/joIx8llqx8abCMhkhgwmxVk8ISSUkknkj/z2QSYsPqu5iZTC/rIPp3Wj1HPCmJxhM4LGZMibD6jFscPPXOp7Cbx9ZgJoR4SUq5IXe7Jd/OZTIHOJpxux04q9A+Usq4EMIH1Gvbn8957Jx8BxFC3AjcCDB//vwxLfT1ra+nLzxyTGA4liAaT+KuKvEf2PYPGNitWsxXv2NMaymLV3+n5hQvz/kyPPddmLVM1YfnY+AQ7HyA+xLnEXOqUsW3r3bByz+HpecqRdJJRob6Ea/8gp2us2iyhqkf2MrQqdfido5e9fP37Z0MRWK8fe08tWHvgxA6BmvfWfoCXrqLqLOBu/qWc+7yxpI6a8fLb188itdp5bLlLXnvj8YT3P3cYVbN9XDWwvq8+5TCzg4/T7f18k9nzad6YJea0XDKNeBq5LcvHsXlDNGTeI79NitrXbIk4/ti54sscC9gnmMdLTt/wnDDGlzLN/J8x/O81PUStFwJyZiqpCtSuXXAd4De4V6uWHgFTdpn8d6Xj3Jl+M9Ym5crJU/gd5uP4qlS79WDB/7Ky3ZfyR7Bzr6d+KN+rllxzZhPlHlJxuG577LbvpoVa5W+UCyR5K5nD7F6roczF9arwoznvwtzlsOC88Z2nFCfUohddhk0LmfHcT/P7NP+n0efUM2TZ1yHSVQ+tVsJQzApSCl/CPwQlEcwlue4Yc0Nebd/+vdb2Xyon8f/vYCMcS67n4OBQYg7YcOnxrKU8nj2FzBr1chjPfY9sC8ovIZt98AzP+ORxIVcuu41fOuRNt5/7WvwPPJtcC6ZnLXncPTZ3zFv4Ds8suafWR7fw9wDT/BI1ZVctGHtqI+979FHOX2ul09tOE1teOl+cC4t73XsfJLksJ8fdF/BKacu5yMblpT0sB8+uZ9zFjWwdu7IOH8xgpE43/nDg1x70TI+sWFpwf22bXuevvYon3r7a8p6/kzufGwfj3Tv4T/OvBzHL98EpkZ47VdACNp2vcSO3jaoew6fycTKutG/QlJK/BE/b17yZq5oehfLnvk6u2dvYMWGT3H7S7dz9867kVYnApQUdRFD4I+qPMkH1n6AJbXqPX/xwbu5KfZTuOBmOFXN6N23+yV2dw7xqQ0bOeI7yFFfe8nJYv0Y/3HGf2AxVe7UFunej/2vX+avi17HFRmftec2P0OyT/Cpt5+rNjz/K0i6x/69OvQMPHonLLsGFr+O2x7ay2M9bfzX2a/H6l0Ov78eWl6j9J0qTCVMyzFgXsbtudq2vPtooSEP0FfiYyec7cd8HB8Ml97qn1k1VEkt8kIE+7JLR3VG6+rUvkC1dQ2c0VqHlLClIww1s8dUORSKxnnd1x/n79tLr/TJpWv/qwCsXLuBplnqX9/ePnqD21A4pjSG9EqXZFL1I4zWUZxL7UJMA4fwOq10+ErLEUTiCb78191846E95R0L1bgmJayZMzI/kMmFK5rY0zVE+8DYdXsGQ1EcVhOOcA8celqFU7Sr/uUtNRzrU3/7TCaWukdPlIfiIeIyjsfuocGi1tUvVZGC2+4mnowzbNFOSqM0lfm1z6JbSwAnk5IFwzvVnRlT5ZY213C4L0g4lsBtr8VnMpfsEfgjfqqt1RU1AgAHD6mEdtOchVnbz2ytZetRX7rooHYh9I8j9JwpL4GaVVzntKkw2cLXqFzK/onJE1TCELwILBVCLBRC2FDJ3wdy9nkAuF77+23Ao1KddR8A3qlVFS0ElgIvVGBNJRNPJDnQEySaSDIUiZf2IP0floxBeHDC1gZoOkO+7GYyndGkqLVkcXNDI6fM82IS8LKeMB6DIWjrCnCwN8hX/75HlSKOgWjXbrpFPbObGrG5Vaikp7N91MdtPjSAlLChVatj97ereOpoGkO51C2E4X4W1STo9JU2l6DTpxqanm7rLbvef/sxPVFc3JN4zTJl6DcdGHvViRKcs2lzBySsTocSV7TUkIyrHJPPbKLVNboh8Gmy1R67Bw/qc9YXVx3dHi1p7DdreY9RSkj1q3W3TRkC33CMdbQRsbigPu0pLW1ykZRwoCeIx+5hyGQq3RBE/al1VZLjR1S1zsKF2d3rZy6sJ5pI8mq7tr7a1tTQnzGREpxT4cGsEZXOOqV0OkH9BOM2BFLKOPBR4EFgF/A7KeUOIcQXhBBXabv9H1AvhNgH/BvpJPEO4HfATuDvwEeklJM6uPRQX4ioVq2Rmss6GqE+cDWrv8uUSCibnA9HFqPMLU6EfMSliTlN9bjsFla0uHn5yEC6hLRM9LK+g71B/vxqeY1gAImkpCZwAF+1dmWleTlDvaM/1/MH+7CaBafN1wyB3pRWrkdQtwiAtVX9dPlL61g9Pqj2iycl/9hZnlrqjuN+6qttNLuLx6xne9VJun8cjWUDoRieKqvKYdUtzsoBqVJRM6akHZ/JxGz76McZjAwCyhCYNaPQFVPr1K/sfWgXBKNUDvkiPuxmOw6tuKInEOFU0z58dadkVcPplUNt3UO47W6GTYLo8OCoa9WPoa+rkgx2HQGgflZr1vYNC9Rn8YWD2ne0bqEyWmOVg9C/61WqoaxnKJKtOrr4Qji2WVUWVZiKZB2klH+VUi6TUi6WUn5J2/Y/UsoHtL/DUsq3SymXSCnPlFIeyHjsl7THLZdS/q0S6ymHzPF4fYESrhBjYfWh12WPJ7oMMyUvoU6aoWicT/zmFQ72BkedWzzk7ydAFQsbVbnl6QtqeeXIIEnvfPAfL7t1v72jg4ftn+Y9tdv59qP7SJbpFew4NkirPI5ZL/d0qddkCfeN+t5vOtDPKXO96cobXWyu1B4CHa2XYJm1hw5fqYZAhZBcdgt/frW8sNiO435Wz/GMmpittpmxmESqKWws+IajNFahwkJLLsq6b0F9NQ6riWS8Cp/JjDM5eslnyiOweVInt46oI70N8AvtMzBKL0Hu1Xp/fz/LxRGizadl7dfa4MRsErR1BdLHCJd2Yp0ojyA60E5U2EY009VW21jW7OKFQ9r6dFHHsYaHQn3qO23VjGXu0Pr174J335vuIaogM76zONMQ9JbiEQxrrnujZggmurtYb17TQkNP7u3l/i3H+eu2jlHnFof8/QxJJ4s0Q3DaAi+BSJwOUwsglY5/GdQf+itLxHE+Zfsj+7qH+Nv28q6Ot+7aTY0YpnHROrXB4SVpstIgfOwsIgIXiMTZdszHWYvSrff07lFyB/lCZsXQvqytpi76ghGi8dFzPHou4R1nzOPZ/aMbLZ1IPMHerqG8/QO5CCHwOq2pUZNjYSAU43SxR52UF2cbArNJsLSphkTCyaDZVFIC1hdNh4Z0Q9AeViemtEegOfCj5Ahyr9bjx17BLCSm+Wdk7We3mFlQ70x5BJnrGHW9E+AR9AUiVEV6GHY05S1RPnNhHS8fHlA9IClRx0NjO1ioLyUvIaWkN5BjCOoWqXGsYy1PLYJhCLqGqNauMktyy3X3LeURTHBoKCUvoU54uhTBtnZNAKzIFzAS9CmPoF5dQZw+X33Itoe08EqZKqTr+/9KHDMe327eWnuAbz/aVpZXcGyfmu1aM0frNxQCnA00UHxYzEuHB0gkZXZpZY+mMVSiaFoKuwtczbQkOpESuodG9wqODYapr7bx1tPmkkhK/r4jwwAOHoHB/Aa1rStAPCnzagzlw1NlxTcOQzAYinFq7GVVVZKnpHhFSw0y4WTQbC0p7q4neL12byoXdmRYnZhSV+uyNEPgj/pT+QEAW8dLALgWnz1i32VNNdkeQbQ0me7cY1SCre2DtIh+ROaIygzOaK0jEImzq2MoQ+Z9HB6B9j0fisSJxJMTPqtYZ8Ybgj2dQ5yxUJ0gS7rS00/M9YvVF27SQkMNSCl5UjcEx3yjhoaSwz6GTdV4NMmBeXVVNLjsPNendeaW8YFN9O5ndWI3z855L1Q38u81D7G7c4iHd5X2+qPxJPHO3epGRlzfVNPEHFuwqPbP8wf6sJgEpy/IEDzr3VN6R3EutQupi6oEdSl5gg7fMLO9VaycVcOixmr+vDUjPHTfh9PTtHJIJ4pLOzl5nbYxh4aklAyGoqwIvgjzz05rUWWwXDMEPnNplTh6jsBtd8PwAEnMHAmY0tsAX1L7zpTpEXj6t3JQtlBT2zRi36XNLg71BXFY1AWMLzZ6GEtKOSEewZajPpoZwFk/N+/9Z2rnjhcO9av3vLpxfKGhjEQxTPysYp0ZbQgi8QSH+kKsme2hxmGhrxyPoLoRXE2TExrSdIb29wQ5NjhMa72TY4PDDJuqis4tFhE/SV2CGhV+OH2Bl8ePqw7FclzYoU0/IykFvhXvhDM+QEv3k7ymto87Hm0rqez21fZB5iXbiVlc6UQ7QHUjsy1DRdVANx3oY+1cT1rCOdin/g/l5gd06hbiCqqr+FLyBMcHh5nlcSCE4Mp1s9l0sC/tSfiOQvfOvI/bcdyPy25hfp2zpGXVOq0MjtEjCEYT1CYHaA61pZqzcjlrYT0mWY3fJEoyBL6IjypLlWrOGh4kYq0hGE0yHE3gtDixCAv+hPadKSdHICWz/NvYbV6RN3eytFlpDvkC6gLGHx/9fxROhIklYxXPEWw5MkCLaQCLN2+fK7M8VcytreJFfWJZ7cJxhoayDcFEj6jUmdGG4ECPUmRc1lJDg8ueEnkqii4q5axXxmDCPYIedSyTKRUW+tBGVcbWOWwpOrfYFg9gcmR/MU5fUMuh/jBx9/zSK4eSSew77+Hp5BpmzVsEZ7wPLA4+1/AE24/5eXzP6Mbwuf19LDEdRzTmhHOqm6jDx4HeIKHoyPLdUDTOq+0+zl6UERbSx1yWWzGkU7sQS7ATO9FUaWgxOgbDqaqeK9fNIilVlzOgQoOhvrxiYzuO+1g1242pRNkIT5VtzIZgIBjlfH2SVQFDsHauhxvPX4NfQLJEQ5AKtQwPELN5ASWGJoTAbXfjS2ifvRKqhlJX676juBP9HHbmSpIp9DGkXYPq9OSTUaXDU4TcPoVKIKXkwNHjOIhCTf6ucFBewYuH+jV139ZxGIJ+wyOYCtLj+lzUV9tKzBH0AkJVELiaJiFH0JeRKO5hUWM1l61WH8ojQe3flydh7A/HqJIh7NXerO16eKXfXkZT2ZHnqAq284fEBSxudKn1nPJOFh3/E6s9Ub71yOhewbP7+1hu7sDSlHPydjXiiqsv0e7OkbHglw4PEE9KzlqYkSjWxebK7SHQqVuIQLLI2jeqIfCHYwxF4sz2qkqOZc01LGt2qfBQJJA2wn37sx6XSEp2dZSWKNbxOq0p4bhy8Q3HeI35VaL2OmhZV3C/OoeXpIBAiYbAa/eqG8MDSK1qRr9gctvc+GMh1ehUpI8glogxHB9OX623vwhAtzt/N/mixmpMAo72qkS+3zR6cltPKFcyR3CoL4Qjon2/C+QIAM5sraMvGGV/T1CVkPraleRGOejViM506SgYhmBS2Ns1hMUkWNTgoq7aVlofQahPjeozWyYvNFTdQDiW4PkDfbx2WSNep415dVUc9Gv/vjxXYwe7A9QQosqdPUhk9WwPNrOJI4kGlSwupZt666+ImKp40XEutdWajMDZH0EkInxl3ia2HB3k6X2FRcfCsQR7jhyjQfaPjOtXN2JKxnATypsn2HSgH7NJsKE1s2JorxKa88wbsX9JeJVW1Vqnj85RcgQdWg/BLE9a8O/KdbN58XB/diNc376sxx3sDTAcS5ScKAbwVlkJRhMlVTLlMhAMc4FpG0NzLshWqc3BY1fr8UVLqxrS9yc8iNAkqPXvidvuVidga/EO99RJWr9ab99MGBuR+pV597dbzLTWV7O/e5gas6MkQ6B7BKn1VoAtRwdoEVppaBFDoOcYXzzUr1UOSVVEUA45/UK9gQgWk8Bbqv7ZOJnhhiBAa0M1NouJepedvmApoaF0Zp9qzSOYSJmJYC84G9h0sJ9IPJnqQF0z28PeAe24eQzB4e5+bCKB21OXtd1hNbNmjpvtw3XqyzVa80s0BDvu51n7BcxuzAjPNC6DpZex9vjvaXULvv3IvoJP8fKRAeYntJNm7lV8tUoWtjpCeSuHnj/Qx5o5HlwZIx7p2QP1S4qe8IqiGYJljoFRk8V6D4EeGgJ4w7pZSAmbtmXkBvrash63XR9WP4q0RNayNCPrGy4/PJTo2EaD8BNrLa6XpZ8o/fHS+ghSJ9bhASyubJ18j82jOoZt1UprqAB6V7HuEcj2F3k1uYh6d+F6+KXNLvZ2D+G2VOEzj95dPBEewZYjg8yzaMd1FzYEixqqaXDZVJ5grJVDufISWjNZqWHF8TLDDcFQSn2ywaVCQ6OWQwZ7012+riaQiYkdLB1UHsGTe3uwWUycrZVQrpnj4VCgcGjoeKeKYXvqRtbZn76glk2D2hdmtPDQ7r9AdIjfRM5VYaFMzv0oItTLl5fs5oVD/Ty7P79X8Nz+Ppaate7h3ASv9sE/rT42opdgOJpga/sgZy/MNmb07h17fgDA1QJmG63m3lGTxcd9uiFIy40vbnSxcpabHXs14ydMWR6BPxzjZ88dwmkzj3zPiqBf/fnGUDnkbn8CAOvSi4rup5/YB+PDo17A5BoCm0urrgtmeAQRnzaToIghyIzfxyPQ8SovJ5fQ6CosUre0qYbDfSHcFpfyCEYZYj8hHkG7j3UeLeRVxCMQQnBGax2bDvanZl6UnSfIlZcIRGioKWEOd4WYsYZgOJrgSH8oNeSkvtpGUsLgaFdjmcOlXVrpW3CC8gTxqNIZcjbwxN4ezlpYl+qsXTvHQ1BqJ6c8HkF3rzopW6tGfjFOm1/LwbhmIEb7wG79NUn3PP4RWsKixpwruNYLoGUtZ3f/mlk1Nt5/92bufGwfkXi2Ssiz+/s4u6ZPldvqV0w62nu4xhtld4c/azjLy0cGiCVkdqI4ElCVOmOtGALlSXjmMZseuv2Rosb/+OAwZpOgqSZ77sSV62Yx1KcZt1mnpHIE/cEo1/5oE6+2+/ja29YpwbAS0UdMjqWprKn7GXYl51PTmL/MUScVGjIJKFKfr5djemweNdci7MdSXUeNw5KKX2d5BEVyBFkeQed2RCLCluSSovHvpc0uEkmJzeTCV4LeUK6W0XiJxBPsOu5nWVVA5QMzZ4Hk4YzWOo4NDtORcIOlqvwS0nw6Q5NUMQQz2BDs6w4gJSmPoF5700ftJQj1QbV2YtLCGhNWOaR9OAZMHvZ1B3jtsrQC6Zo5HgJoH848ekN9fdrVeZ4qitMW1HJUas9VzBD4O+DAY3QtvBqJaeTVrRBwzscw9e7lgdeHOX9JA//74B4uve1JHt7ZhZSSYCTO1qODrHV0qd4Lc44ypPYeLnGGiMSTSjpDY9OBPkwiQ2gO0iGYsfYQ6Hjn05joIppI0l8kQdsxGKbF7RgxMObKdbNoFD4kAuadDX376fKFeMcPnmNP1xA/fM/pXLludnlL0iaLlV05FAkwy7+V58R6bJbiX2k9PDPayTVTeVTtJ6GqlgaXPcsjCEQDJKzFp5TpUhVuuzuVKH5lNEOgqcwKqXsEo4SGIj7MwozLWroHVoydx/1EE0nmWAaLegM6S7RKp6MD4bFVDhUTnJsEZqwh2KNXDGlzW+u1+GxRmQkps2p908JzE5Qw1hLR2wfUleJrMgxBXbWNGrdX3chxy6WUDA1qHyzHSEPQ7HZQW1vHkNlbvIR02+9AJtladznASI8AYPWboWYWjdt+xA/fs4GfvfdMLCbB+3+2mRt++iK/23yUeFIyN340f5WPsx4QzLWpE0lmnuD5g/2smeOhxpGRMBurxlAu3vm4w+qKvljl0HHfcFZYSGdBfTXLqofxCbcKU8WH+cj3/szxwWHu+uczuHBFc55nG2VJmkdQduXQoacxyzhbbKePumuqEcxcPNySqTyaUth1eGlw2ejN8AgkkoC1qmgfQZZH0P4iIUczXdTR6Co83U+vHIrGq/GXIInhj/qpsdWUNGynFLYeHQSgLtlXtHRUp047f/QHIyo8NKYcgYCqWpJJSV8wahiCyaCtawib2cQCrdkn5REUSxhH/Ep6OmUItBPzhHkE6qp+U5dglseRqq/WaZ2leSQ5V2Nd/gi2hGYcCtRVn76glsPJRmShKxcpYcuvYe6ZbB1uwGoWzMvXGGWxwVn/AgefgM5tvGZZI3//19fw2Tes5OXDA3z+TztxmhNUBY7kNwRmCzjrqMWHzWJK5QnCsQRbjgxml42C6iEQ5pSK6Jjxzsce6cNBpLghGAxnVQxlsqJmmM6Em1eGVZitPnyYX7z/LM5dXKb+kb4kp54jKNMj2P8IEWHnaM3ow32sJivVZseoHkGWIdBzYFW11Fen+210o+K32oqGhvTnqrHVQPuLHK9ZA1A0Bu6wmllQX40/7MRvMiFHUdz0R/wVrhgapKnGjjXYBe7RPbt6Ld/RF4ymPYJyxgCH+pR2lsnMQChKIimN0NBksKdriMVNLixaDDf1jyzmEeRKQju8ar7oROUIgup4Tx6D1y5rHHG1s2SuuuqMBLO/0Ad6AtQI7YuZ0VmcyekLajkQbyDReyD/B7ZjK/TsglPeyYGeAPPrnIXj3affoEoIH78VpMRqNvH+Cxbx6Kc3cu1Z8/nkaRaETBRO8FY3YQ72sKKlJtVh/MqRQaKJZHZ+AFTFUN2iotOwSsK7AIA5ordgCWkyKen0hbMqhjKZbRmiR3q48S9qzZ8/z86p82vz7lsKLrsFs0kwUK5HsP9RtlvXUu0sLSzitbpGHfiSkqC2ZRuChhpbKjSUCjNZbcWTxVE/NdYazKF+GDzMAfsqqm1mnLbiA2QWN7rwhWzEhSCkiz0WOUYlK4b2dgVYPasaEegqzyMIRFUJaSxUXo9RZlexZmgbDI9g4mnrCrCsOf3FqXXaEILiMhOprmLtik8IrYR0gkJDmkdwJOLMCgvprJ5Xz7C00def/SU50BukBm36Vp7QEKiE8QE5C4v/CNx5Jjz2Zejend5h62+UkVv9Zvb3BItXv1TVwmv/HXb/GXbcm9rcWGPnS29eywdWal2hDQVGNVY3QLCX1bPd7DzuR0rJ8wf6EILs/gEYf8WQjlZCusDUW7CEtDcYIZpI5g0NAdjDPeBqxOpuIWl10RIbfcBOMYRQdeNl5QgGDkPfPp5jfcqjGA233a1CQ0XCLdnKo4NqY5WX+mo7A6Eo8UQy7RGYLEXLR1Ndxcc2A7BdLCsp7LG4qZp+vyqO8IeLG4KsLuhxIqXkaH+IVe6oqgosIUdgt5hx2S1pjwDKCw/l0xkyPIKJZSgc49jgcGoIBiiZ3lqnrXiyOKUEmnGV6ppAmYlgD0nMBEwuzlsyMtywZraHAA58vhxD0BOk1qwZggKhoRUtNfzc9Cb+PPfTKtfxxNfgu2fBnWfD41+Fbb+H5a8nbvdyuC/I4qZRrjbP+RjMPg3++umRhlGXhKgvYAhcTRDsZtUsNwOhGB2+MJsO9rFqllsNWtFJxKD/wNg7ijOpVR7BCsdAwRJSvZlsdr7QkJQQ6OHstat49N9fh6lhyYimsrHgcVpHr1zLZP8jADwaW12yIfDYa0cNDWUpj2Z5BHakhP5QND1tzGwa1SNw27REscnCK/H5pRmCRhdxfaraKDMJ/FF/xeQlBkOqm3xJlVaEUYIhAOUVDISiYyshnUJ5CZihhqCtW8XUlzdnh03qR+suToWGMq5Sq5smMDTUi1/UcMq8uuwTokZjjZ2wcBIcyv5CH+wNMKcqptRJTea8T20xm1g+r4XvD2+EG/4Mn9oDr/9fdXX/+FeUN3LKuzg6MEwsIVnUMMowDLMFrv6uqmD666ez7+ttA/fcvIqYgKbZ1MMqrQv3lSODvHJkcGRYqP8AJOOV8Qiqm8BsZ7Gt8KQyvZlsVj6PIBqA+DBWTzMOq1l5OzlNZWOh1mkrT4p6/6NI91y2hJvUmMoS8FbVl5wjUMqjg2qjw5uq/e8diqab00wC4mFVZlrguVIVQy1rOR4o7SS3uNGFTChD4I+NLjFRKY/gSL8Kq7batGMWaSbLpE6XqfHOB0R5JaTB3tR5Rc/BGIZggtnbqWsM5RgCl614sjin+w+YUL2hqL+bzkRNVtloLtJWTSSU/SU50BukxR4tmB/Q2bCgll0dQ6pKpaYZzroR3vs3+LedcO09sOwy9mtGc1SPANSMhtf+J+y8LytERM8oktHVjRAdYkW9BSHg1y8cIRJPjkwUj1djKBOTCbzzWGAq3FR23FfEI9D/53oJcf0SNZegzKlvuXirrKXnCBJxOPAksdaNJKXIe7GQD4+jdlQp6sHIYFp5NDyockAWW1ZRRcojSI2rzO8V+KN+3NYaOPYyzD2j5Br5xY3VaUNQTG5dJhmKDlUsWXxYMwSzTINqQ4keQepC0mIH95zSQ0M51Yg9QxEcVlN2N/0EMzMNQVeAKquZubXZX/D6jBrpvIT6VNzclnFSdDUpaz4BMhND/Z30y5q8+QEds6MGEQ0QiKg4fDSe5Gh/iHprpGBYSOeilc0kkpJHduUYMvdsWHoJCMGBXs0QNJRYn33ev8Ks9fCXT6v3RUrlERQ7eWtNZdXxARY2VPP0vl6ESGu9p+itoCEA8M5nluyiq5AhGBymymrOH3LRDYErwxAgldcyDjzlSFEf2wwRH77ZFwCU7BG4bW58JkGySCVOrvIoVSoJrssi9wYi2Mw2qixV+NE++wVKSH0RHx4JRAPEZp2GPxwvSV7Z67Th1Rvg4oWrkgKxAEmZrJhHcFQzBPXJPtU1rhv7UajLFK6sK0OOOjKkqhFz5CUqVQpbCjPLEBx8Eo48z96uIZY2u0boeDSMGhrS5CUy/0GuZk1mongyaywkAj0MmT2snVP4Ssde7aaaMLu0sssj/SGSErymcMFEsc66uR5meRzZE7dy2N8dpMFlSw23GRU9RBT2wV//HfzHVCKx2Mm7Wi/D7WHVLLXmFS1uvLkntp69xUNM5eKdT12sk6FIPGVIM+nwDTPL68j/hdTzQlmGgHHnCbxVttLLR/c/CsJEZ8M5ANRWl5oj8JAUgkCRBKwvmqk8OqiEFklX1/UOqe9Jja0GH9p7l8cjkFIqjyChwkYDDpWkLzXssahOfTaKzSSotLzEkb4QDS471lCnFkIs7cpcNwRKjnpB6aGhnGrEweFYyUa9UswsQ/DXf4enb2dP19CIsBBAXbUd33CssPpjpryETuokVtmEcTIpcUQHcHhbRnS1ZuKq8eJiWI2uRJWOArgIjeoRCCG4bHULT+7tyTsLAOBAb4BFpXoDOs2rVYhoxx/hya+rbUUNgS7V0cNqLU8wIiwE45tKlg/vfKpiA1QRzttLcGwwzJwCpaMp1dnqXEMwvjxBrdNKIBInlijBwzz8LMw6hT4tfOKpKjFHoJ3gfVqJaD5ydYZ0j6DGbsFmMdGrhVA9dg9+WdgQDMeHiSfjeLT8QY9Uz1mqIVjSWIdJCnzJaMG6/ErLSxzuD7Kg3glDnSWVjurUVduIJpLqoqJ2ocodjjK5DciecQIEI3Gq7flzexPFzDIETatIdO2kZygyIlEM6audgjHazK5iHf2KsMJ5gl3tvbgJ0tBcvJnFUe3BZYqwXau/1yUaHInAqB4BwKWrm4nEkzxRYLjM/p4gi5tGSRTn4/x/Vbr4L/1U3S6W4NVzLsFu1s1VJ4pzF+e8z8mkFmKqQKJYJ6OXIF/CuEObTJaXQLcWNtDWbnepWHLOXIIRxMKw476CJ7WSm8qkhK7t0LIuFUqqLblqaPRZwCMNgRdQFw8N1baUR+C2ufElCo+rTJ2kY2r/zoT6TJZsCJpqsCYs+E35n19fa+brGi9H+4fVVDl/R0nNZDrp7uLMEtJDoz8wlF2NGIwkqB6lx6LSzCxD0LwKs+8w1QynxOYyaRitqSxTeVQnJTNRWUOw77CSfmiZVVxEDJuLGhFOzcc90KNCOeZoYNRkMaihGrVOKw/mCQ8NBKP0B6PlewQAZitc/T0wWcDhSXtO+XClPYJzF9fzy/efxSWrciQaBg6qGHRz/qlWY0IzBHNFz4iEcTSepCcQKdhMRqBLmxyXceVWv0QZq2K8/DP4/fXQsSXv3R6nrjc0SsJ4qEOdoJvXpPYdEUorQEp4LlaiIQgPquZJjYaatGS7x+bBn9Deuzy9BKmTdCwMDg9dWqi/1BGMixqrsSStqsqpQN9DJT2CaDzJcd+w6qIf6ijLI8jqLtZLSEsJD+VUIwaj8fRY1kliZhmCptUALBPtLG/J5xGMIjORzyPQT3AVLiEN9qtQU03dKB9EuwuHDLOve4hQNM7B3iALG6rVl6aEumqL2cTFK5t5ZFf3iJBYKlE8Fo8AoGUNvOGbqsegWOLLWgW2Ggj0IITgvCUNI+PyHVvV71mnjG0t+dCayubm8Qi6/GGkLFAxBCo05MoxVvUl9BK0Pah+FzAYuhT1qAnjrh3qd/PqlFppyVVDNl2KOn8CVkqphtLYRoaGgKyxrm67G19c61kp5hGEA+BqTj2uvogEdSaLG12YEnalN1SgyilV6loBQ3BscBgpYaHHrPJ+NeV4BOr8keouhhI9At0QKO9ShYYMQzBxaFeTp9iO0eIe6fLrrl1ejyARV1dGmaWjoK52zfaKewQRvzIENvco4mU2FyaS2GWEXR1DHOgNsKTeoa6eHaW5ypevaWEoEh8xT2B/t/pil6OpP4LTr1ddx6NR3VDcmHZsVTLWjfmnWo0JVxNYHCy19tKhzR3QOZZnIE0Wge6RXk7DUnXyyDO/GFB6PAefUn8XMBhp4bnRDMF29bt5FYOhKG6HpWguKZOURxAP5w1RheIhFde3eyA2rHoEtNAQZPfbeGwehuLB9OvLIZXIHfaBq5meoQhepxW7pbQY+NxaJ6ZkVVFJjJSoXcc2+NqicX0XD/ep17KoSitXLccj0ENDoagynHZPaSWkIU2iXfPgA5E4LiNHMIF45jMsqjjD2Zm3EqShOl0aNwK9uzLXIxBiQnoJkpohKBpSgVQFjYswz+3vpTcQZVmt9uUusdPyvCUNVNvMPLgjO+G9vzeAzWxibm0esblKM9rYz85XVZ/CeDWGMhECPPNYZO2n05f9P9cNQ95mMlD/bz2kpTNa5dDBJyERAURBj0CvFhm1u7hrh6qgqqpVVSbVpb8vKQVSE+pEn0O24Nyg2pjpEdTY6Quo6hi33c1wIkIU8kpRpyaHBftThqAc6QSzSeAw1xQdTuOP+LGZbDg6t6uT6sEnS37+XPTS0VImk+WSlSMQAupaS/cItGrERFISjiVH1WGqNDPKEEghaJNzWWE6mvd+d5UFi0nkH2KfSujkqWapbqx4aKg6eIQkplT4oiA2dRUxpzrBA1uVrPJitxbiKSFHAErpceOKJh7a2UkiY0jL/u4grQ3Okq80x4XWXZwXKZVHMKvwUPYx453PHNFDpz/7hHh8NHmJYBFDUChP0PYP1ZjVen5BY+EpVYq6a4eqzkINsilntq3VZKXaZCvYXVxIeVSnXquO8Q/HU+Ejv9mUt48g5REEVCitJ1C+zr7L5i06k8Af1ZRHh7Q816Gny3r+TI70h7BbTHjj2ve9xGYyAKfNjN1iSp8/altLyxEE0yHnoFa9N5nNZDDDDEFPIML2+FzmRA/mdYmFEKq7OF9oKFd5NBNXc8WF5xrDh+i3zgJrYc12QE2HAtY2Wtjbpa7IWl1aq38JVUM6l69uoTcQ5eUjaU2XMZWOjpXqxsIegf+4ev9nra/8cb3zaUp0jfAIjg8OU+u0pibCZRHxq3BJbqORd4Fy8fOd5KWEtodg0UZoXqOqi/J8Bms0BdKioaF4VInvaYbAF4qWnChOLdXiLBhuSQnO2bJnEejoJ/LeYCRDeC6/3pAvqgbGOCMBcDWlmqXKWquzgSGziUiBkFuq+W1Imxh3+Nmynj+Tw30h5tc5MQU0o1KGIRBCZMvU1C5UQ+wLSG+kCPWlE8VaP4uRI5hA2roC7JHzccQGC9b911cXGGKfk9DJosLCc1JK5iTaGahuHX1nLTS0ul5dtZsESl4CSg4NAWxc3ojNbOLv29UXIJZIcqQvNPZEcbm4mtR7nMjTz6AnilsmxiNwJXyEAr6sZHlHEfnplNHPTRabLapaJF8vQc9u8B1RHdv1i1WFzVDHiN2EUFIRg8XmFvfuVZpLmR5BqQ1/Gm6rq6ACaUqCuqBHoBmCoUhairrA3GJ/xI/H6kJAKllcrkfQ7FVx+uN9I98vyOMR9O4Z84XZkX5lCBg6rnJ/Ga+7FGqrbWo4DajPQjKmmiqLkVGEEowoo2H0EUwgezqH2Cu1cky96iKHepct/5SyfMqjOtVNKnQ0muUvkcFAmIV0MOxZPPrOWmhoqVcZgnl1TmwxLVZbhkdQ47By/tIGHtzRiZSSw30h4kk5uR4BMm1wM+l8FRCqCqnS1KZ7CbqH0pVDxweHCw6kSYUBXXnyN/VL8vcStP1D/V56yai5BO9oMhOpiiH1fgyGomV3onps7oKhofzKo97U/fpAmb5gNGM4jaOgR+A2K682bG8gFE2UbQjma5VzxwfzX2yl1E39HVCnfWcOP1PWMSAtP61KR7VmsjJlHrJkJvRegtHCQ6G+VBFKyiMwcgQTR1v3EF0O7YPSvTPvPvXVBYTnUt1/eXIErmaQyfwnMZ1gX/pLNQr9HfuwixjJugKyzZloHkFrjbqaVaWjWn14mbK8l61upn1gmB3H/akO5ZLE5ipBqgw3z5Vcx1ZVkWObAO9E6yWYJ7qzSkiPD+YfUQmkvb98GjS6Ici9KGh7SJ24PXPTcxmKlJAWbSjr2q40r+qXEE8k8YfjJZeO6ngc3lFzBFnKoznlo6CKKtIegaNgjsBt0gyH8ALl6+wvrlfvc18w//crpW461AHLXw9W55gMQX8wSjCaUF3FvnYlHFcm6vyRERqC4gnjZEKdFzK6isEIDU0oezqHaGyerU7cXQUMgcuu6oBzCfWpE6slz4c4NbKySML4N9fAH28saZ3Dx3YBYG0uoYtWE8Crs8ZorXdy6rzatLtfpiG4eGUzJgH/2NHJ/h6tjC7fnOKJINVUluc97Hi1sv0DmaR6CdJNZYFIHH84Xn5oCJQhSETUiUQn7IMjzylvAFRtuqWqYBey12krrkDatQMaV4DZgj+sThyldhWnjuGo1+YWD464zxfxpZVHhwfUaNCMz1Kt04ZJqPneKY/Akn9KmS/qw62dZrrLlJfQaXYpIzRQQBLDH/XjFjYVhvHMg3lnwqHyDYGuOjq/zqnCbw1Lyn6Oump72iNwz1ENlcVKSIcHAZkyBLrmlZEsnkBOnV+rOlabVkF34dBQMJpgOJpzRZeR0BmBfkIoVDkU9ikt9vYXS5pjmtAGtNfMKaGLVrtKFtEAf//X1/DRC5ekr/LKCA2BMoJnLqzj7zs6OdAToLHGjttR3glmzKQ8guxeBoJ94G+fmPyAdlxpcTBX9Kb0hjr0OQQF5SW6lLxEvs+DfrWfmSfY/5iK6S+9VN02mYo2n406paxrRyosNFBmV7GO29mAz2QiOTzSIxiMDKabs8KDqh8lI0RiNgnqqm30BiK4rC4EAp/FWjhHICUIMx0xVYZcbrJYX0swNrI8NZaMEYwFcetfK/csWHC++n4X6ucogF46urBqWH3fxyBnUu+yEYomCMcSKmfknV88NJQjLxHSzjtOI0cwcfy/K1fx/gsWqSRbz568MX29KWREeEhXHs2HHiIo5BEcfUGFjoYHVAXMKFgH2uiVbhqaS6hY0CWxIwEcVrMq9Yz4VaIrn/cyCpetbmFvV4An23pYPFneAGSI9+W8h50T0FGciRDgnc8Cc0/KEOjNZIUF57RmsnxDf1Lx/4yr/baH1Ml07pkZ+y0uKFDnLTacJtgLgc5Uolg3GOUmiz1V9UqBdLh3xH3ZyqMDeROm9dV2+gIRzCYzLpsLv9mcNzTki/pwx+NQ3UhvUF3tlusR6A1wITmslD0zGNL0knRRO2pmQet56u8jz5V1nCN9av1zElp5eeOKsh4POb0EoMJDxUJDOfIShkcwmTStUuV/ebTj9YqIESWk+eQldEYLDWXGK/WO0CK4/Ac4xJzSmkrMFhVmyBQQC/tL7iHI5bLVKjHX5Y+waDwdxeXi8Ki4d65XlaoYWjthhxbe+bSa+1JD7PUQ0axioaFCGvXVjaqjVI//J5Ow7yFYfFG2nHH9EjVvOD4yBOR1WhkqpECaIS0BlK0zpJPqLs4jn+6P+PMqj2bSUJMuqvDYPCrfkOMRJJIJAtEAnlg0VTpqEumTZanoHkFMRFKD3XVS+YyYtr2mBeacDhZH2eGhw/0hmt127APa/24MSrcjDUFr8dBQHnkJMHIEk4MuXJanckjXQBnRVBbqz186ClruwFG4hPTws2lphM5XR11e3fAhOqyjNJJlHd8FkQy3OTJUdlhIZ7a3KqUAOi5piXIRQuslyLlC7diq3OtCYblK4J3PLNIewfHBYUwCmgtduQa68lcMgXod9YvTYZ/OrWp/PSyk07BUzbHIc7VYVIF0hCEoT3k0dQztit+fJ+4+GBnMMASDWRVDOvXV2XpDfsEIQxCIBZBI3NFQqqu43mUvu0HRZrZhx0TUHOdAT/YxUvISUa0h0NWiPOG5Z8Dh8hrLUqWjPXtVwtk9iuBjHtIRhYwBNWFf4UKRnP4k3RA4rUZoaOJpXKFivHkqhzIrIrLImCk6AiG02cV5Kl6iITWib9llqkKlcxSPINiHK+mnv2pBKa9EYXNlfwlLFJwrhO4VTFqiWKe6caRX1fHqxOUHdLwLcCf9+Hzqy3p8MEyz24HFXODrkU9wLpOGpWlD0PYQIGDJxdn7FCkh9RQTnuvaod4nLbk+1hxByiOI5q8aShmCYC9Ujfzc12aUuLptbnwkRxgCvQzVHR6CmvLlJTJxCxsRc4L9Pdl5gqxjOBvSEiQLzoXObUXHceZytD/E/Lpq1YfQsFTlcsok7RFo54/RSkhHKI8mcNrMI4ZmTTTjMgRCiDohxENCiDbtd97uCyHE9do+bUKI67VtTiHEX4QQu4UQO4QQt45nLWVhrYK6RUU9gqyRldEQxIcLh4ZAayrLExo6tllVMyw4T4U3OrcVX1uvShSH3CX0EOjYXNk6L2H/mD0CgHecMY/3nLMg/3CYiSRXqiPsh/79E9NRnIlWOWQfaieZlFrpaIGwkJTqCr+YBlT9EvAdVTo+bf+AOaeN9CDqtf9vHkOgn9R9+ZrKuranvAG1TwyTUB3J5ZBSIM2ZSZClPBqPqkS9fjLLWaM/HCORlGo4DYkROYJUh/LwYEpeomGMA9m9FgdBExzsHMx/jNBAti7QgvNUXu7I8yU9fziWoNMfTnsEY5x7MSK0rJeQFhphGuxTsiNW9XmbCuVRGL9HcBPwiJRyKfCIdjsLIUQd8DngLOBM4HMZBuPrUsoVwKnAeUKI149zPaXTtCqvR+C0WXBYTfRlegT5htbn4mrObwgOPwsImH+WMgT9B4pOLZLagPZ4XRmla3ZXuncAxu0RNLjsfOFNayZd+Co1/1lHz6dMhMZQJlovQbPsoj8UVSMqC1UMhX2QiBb3CPSr/aMvQPvmkWEhUHF3Z0PehHFtIQXSZEJ1KDenG+sGNHmJcq8gPQVmAacmitk9Sh5BJtVFU541SqkMkdvmVlPKosGsqrjU1Xo8rrqKx+MRWJz4TCY6u7PDr6ljBHqz5SDmnqHkPkrsJ2gfUPLTi9xJZfyKDVIqQo2mApsKLTcsVfmvvQ/mf0BO7lEpj558huBNwN3a33cDV+fZ5zLgISllv5RyAHgIuFxKGZJSPgYgpYwCLwPlB+XGSvNq5a7lOSkrmYmMq7FiOkM6hYTnDj+jDIDDo32BZcEeBoBo127C0oqtvtzQUE6OYByGYMrQ9Yb0k8lEzCDIR8Zcgk5fmOO+EkZU5grOZaIbgk0/AGR+Q6Dvl6eXwFulT8rLMQT9B1SRQ4ZHUK7gnE5KgTQRztqeJS+hX8XmMQTejAE6HrsHfzKKRGapmaau1pMJZHXTmATndDy2GvxmE3192d+xlLqpvytbMtrmVEnjEhPGeunoEpNW1TdGQ2AyCWqdGd3FFjuseRvs+lP+MFVOWXpICw1NNuM1BM1SSl0ApBPId5k0B8iU+2zXtqUQQniBN6K8irwIIW4UQmwWQmzu6amAwFvTKkCqK6wcGnKF53JqffPialb/1MyS1HgUjr6o1CYhXflSJGEc79rDATmbZk8Z0s+5yeJxhoamjOpGdbWtf2E6XlXvaxma8GM7bgNJs4O5oocdx5XmUNEeAn2thdDDPnv+qvYrFNpqyD/RrKACaWoGQUZoaAw6Q6ApkAqLmgWcQZbyaFFDoI45EFIeQRxJSIisC6vU1XoyScBaRywhx2wI3JokRsjfr2r0M45Rba3GEuwZOUSm9Tw4/kr2d6MARzRDMCeunarGMRI1q7sYYP21KrS8476RO2fIS4DyCE7I0JAQ4mEhxPY8P2/K3E+qAt/Ru6VGPr8F+DVwh5SyQCANpJQ/lFJukFJuaGwcRaO/FPQvU56r83pXjvBcznDpvLialBudGdro2KI+AAvOVbe981VpYZESUnP/PvbLWTSV84XJTBYnk+MODU0ZGSMrAeURTHSiGEAIkp75zBU9vHx4EChWOqrrDBUJDdmqNXkCCUsuKZx0rF+ivMicK8UauwWTyFM11LVDdflmnKQGxqA8quMx2/Ehs0pYs5RH+w8oLas8IdHaHI8ANAXSjHGVqav1ZJI+vEB6HGzZa7V7GTKZcBFKzeUGTXDOUg3IkRcMC85VlVlHN436/If7QlRZzdQMHVDdwPqoyTFQV21jINMQzDlN/c+2/GrkzjmhoeCJGhqSUl4spVyT5+d+oEsIMQtA+52vkP4YMC/j9lxtm84PgTYp5e1jfhVjobZV1d/nyRPUVed6BCWGhiA7PKTros8/R/0WQhmgQpVDsWHsgaPsS86hyV2uIdCueqIBQI65j2BKyWwqi4WVtzbRYSENc90C5onelAx3wdBQyhAUCQ1BOjy0rEBYKHOfnPCQySTwOm0jcwRdO1TMOUOafHCMHgGAx+IcoUCa5REMHFQnxDzCa7oh0D0C0GYS5HgEDmHGLqFLeoHydYZ03FV1DJtMuMRQVuWQP+JPidqNGDQ/7yxlOEvIE+ilo6J3rxKuM4+9o77OZcsuPxcC1r8Ljj4/MhQY6h9hCE5Ij2AUHgCu1/6+Hrg/zz4PApcKIWq1JPGl2jaEELcAHuBfx7mO8jGZoWlFwcohfQIToK7yhSlLk30EqSH2Gcmsw8+qUtXMK6qWteqYyTzNQn37EUj2y9k01owyhyATu2YIpEwnjU/W0BAoj6B7h7qam+hEsYbwzmeeqYd92kmmYGgo2K1OLnlKKrNoWKauLBe9rvA+9bocRZ7KoSrryCllORVDoK7I9ZxCuXgsrhHCcyNCQwWujDPDV6nEs8mUNa7SF/XhFlawOumLqv3rxugRuKvUybLK7E+NUE0fQztx5noE9hqYvb6k+QRH+0PMr3eq0tExNJJlMiI0BLDuHeockukVxCOqETQjRxCMJqg+CXMEtwKXCCHagIu12wghNgghfgwgpewHvgi8qP18QUrZL4SYC/w3sAp4WQixRQjx/nGupzyaVufvJai2E00kU+3ehPrUF79YXbF+hagLkiUTqnRNDwvptKxR7nO+bkOtdPSIaS5uRxlXBTaXCkvFQmMWnDshyAwNdWh5lEnyCKhdgIcALqkmVBXsftVnFY9WY37Bv8F1f8zbjJWibqE6OeTrJXBas3MEYZ+q4skwBNF4kmA0UXYzWeoYKSnqwdS2lCGwVKvO5zz5ASA1I3kw0yMwmbKKFvwRP27UKNc+zbupG2sYq7pZe/wwB3qzj+FB81jyDZFZcB4ceynvSE4dKSVH+kMs9FqU8RuDtEQmddU2fMOx7M5w9yzVXb711+k8Yp5Iw0npEUgp+6SUF0kpl2ohpH5t+2Yp5fsz9vuJlHKJ9vNTbVu7lFJIKVdKKddrPz8e38spk+ZV6qSTM8Qi1Uugh4dyEjp5yQ0NdW5T1n7Bedn7pRLGefoJeveSRBCsWZB3pnJB7Gm9odRc15PRI6iqA4RmCLaqSitvGdVT40GrHJojepntrSr8/ge6C3cVZ+KeDYteW3wfi10dt5BHkBka6laKtJmlo/rwmjGHhhxeTYE02yOoslRhD/ao/pcChkAIgbfKykCuRxDL9gg8ySS4mlMx87HmM9zORu3xw9mhoagfdyKpvK98nf+t56sChPYXCz53TyDCcCzBanuPuqAaR6IY0t3FIxRk179LDanRZyrnyEskk5JQNHHyGYKTniZNaiJHibRei2OmEsbFdIZ07DUq56DHkHV3VM8P6DSuVKGFAoagx9yM1+0p51WkhtMQDZzcHoHZot7nQHc6UVzmYJAxkyFHXXAOAWizioskisulPn/lUK3Tlj2lrEDFEIz95Opx1GkKpIOpbSnl0SIVQzr6AJ2COYKoX+shaKI/GKXGbsFmGdspx+NQrUd22zD7u4MktdnavogPdzyqpCXyeWnzzgJE0TJSvXR0cap0dHyhoTqtqWyETM3yK9TFjR4eyvEIQlo1lGuSlUdhphuCApVDukXvzfQIRtO6ESK7u/jwMyoh7ckZbmF1qIRfvsqh3r0cYk55FUOQHthyshsCUOGhoQ6VR5mssBCkPI+5oqfwZDLQQkOjJIrLQe8lyFHV9OROKevaoU4iGcNSBsaoPJo+RoNSIA2lPeKU8mhJhkAZqypLFRZhGSE854/48UTD4GqhPxgdc34A0sJzZkuYYa0LOBwPE01GcUeHs7uKM6nyKi+8SMJYLx2dHTsCiHTuZozUVqv/x4i5JlZHdk9BIZ2hyW7kZKYbAleTcstGeAQ5wnOleASgdRd3qS/14WdHhoV08klNJJPQu4898Raa3WUkimH6hIZAheCOPKeGu0ymIXDWEzdXaR5BEXmJYM/oFUPlUL8k7/xib5WNoXCcuB5n1mcQZHhIeg6h3DGVOh6neh2+DEOQUh7tP6g8XFfhHo5ap5WBYAwhBG5bzYgB9v6ID3dsWIWGxjBOM2utWvgpaVKx/v09gXQ+Ixwo3mvSer4KDcXzTB5ElY4KAZ7AAfDOU81o4yAlM5HrEUB2T0Ew2xBMlQQ1zHRDACpPkOMR6InCvkBEnaCLKY9mogvP9eyB4f7ChqB5jYoVZg7O8B2F+DA747PKb7qZLqEhUO+hHrOejB4CHSGIuuYyV/Qyu6C8xKAmL1FhQwAj8gRZCqRS60YfUTGkPIJyx1SmjqElYP3D6RGQKcE5vWKoSFJclbhqUtR2T1aOIJqIMpwI40kkU6GhcuWnM3FZ1cVOGHUyP9ATTCmPuocH8ieKdRacpzqyj72U9+4j/SFmuR2Y+9rGnSiG9Pkj75S5zJ4C3SPQZL6nSoIaDEOgKod6dmeVc9otZmocFhUaCg+qMsaSPAItNKS7obkVQzr6EPbM8JAWJ96fnF1+aCjlEQwpj0CYJma+72Sgn2QtVelpX5OEraGVVVUDnL+0gNHXw36VDA2lJprlNwSDwzFVLRQdGmkItBxC7RhPsB6tHNqXIUWdzhEcTAumFSCzxNVt9+A3W1JVQ6mTtJYsHq8hMJvM1GAimAxTY7fk8QiKGILW81Qyee/f8959tD/E/Fq7Np5yfPkBSGtFjZhpAtk9Be0vqJJ0bU5FMKJyBNVGjmAKaF6lrmJyyjlTtcCldBXr6DITB59Q7e55VBuB9JVuZnhIKx3dL2fTVG5oSJ9SFg1qXcU1k5dkrTR6dVbLmvwTwCYQS10r8009zK0tEBootZmsHPT5xb25hkDv3I2le12asg3BQCiGxSTGXHeeUiDVTqi68qjXltFMVoTaajWWMRJPKI/AYkn1EejyEp5kElkBjwDALaz4kjEWNbnY3xPINjbFDEFVLSx8Ley8P++o2CP9IdbX+FU4cowaQ5lYzCa8TuvIZLGO3lOw/9GsasSUR2DkCKYA/cuV009Q77IrTfGU8mgJhqC6EZBKg37BuYVPxq4mdVWZ2WHcu4eozUM/NeNMFg8pGYuTFf1qezLzAzre+doQkcH89wcnwBCYTNmDbPSlVOmhoWiGIViZtY/qKraVV2qcQUp4TpOiTimPIlQopUiiGDK8Fq1yyJ8RGsqUlwjb64nEk+PKEQC4zXb8Msbixmr2dqU9AmUIRtGjWvUmNQQoJzcXjiXo8kdYY+tUG8ZZOqpTV20rbAj0ngLI7iGIGqGhqaNpBSDyVg71BaKlCc7p6CeIWKhwWEinZS10ZXoEbQw4FwJiDIYgJ1l8siaKId2PMZn5AR2thBTf0fz3l6IzNBbql4yQo06JugVjKoRYuzAdAtQYDEXHXDEEI6WoU6GWqKZIOoohSMtMaAqkJpEODekeQSJJH+o49eP0CDyWKnwk2TDfS89QhAN9PaljjJCXyGXFG9RV+M5s8YPUwHra1YZxlo7qqIhC/uQ0oMJDMEKCGoxk8dRgq1YhnDy9BL2BaGk6QzqZJ4hCiWKdljXQvTst+NW7l07rfCyajG1ZZM4tPlkF53TmblBXS4WkmyeSWq15rXukIi2gDIHJUlxqZCzkmV+sy0b4QmGloJmTHwDY1x1gQd3YK1ysJivVmPAlVCVOSoJaV+sczSOoyvYIhgQkIqpqKOUR2Gro1+zKWHMZOm5LNX6TiY0LVVXXjq4uTAiqZR7BuVyqG1T1UE54SC8dbYkeUd/fPPOZx0JRjwBUT0FVXVY5cEjLETiNHMEU0bwajr2SJSFdX22jPxghGSzDEOhXs8760WONzWtV52bvXpWHCPZwWMyhscY+tjF1uhR12HdyCs7pVDfAu/9YuC58ImlZp76YW3+d//5S5SXKRZ9fPHg4tanGoRRIl7X9WG1f/eashwQjcfb1BFg7d3xhQI+wpqSoU8qjwQE11MVTfDxI7kwCgKGYCjOlPIKqxtQJsa567N4LpGcSzLZHWNrk4kBfN25hwWStLu3iZ+VVyvPKkJ4/3KcMgTtwoCKJYp26antxQ2B1wI2PwYWfTW0KGDmCKWbNW8F3BLbdk9pU77KRlBDx96ir7VKqcHSPYP45oydrM6UmtIqhvWOpGNLRpajHMbh+xmMyw6nvVkm8gcMj7w92VzY/oKOXkGZ0GJtMgnMcRzjnyA9h9VvUZzSDnR1+pIR14zUEZjs+qSp/UqGhoS7lJY+SrNcbp7IUSOPZHoHL1ZQqoxx3jsDmwW8yIYcH2bi8ke7gIDWYlDdQSp5k5RsBATsfSG060h/CZTdj6W+rSKJYp67aykAoluqAzktta5YWVTASp8pqxjzJ84oBJt/0nIisuhpm3Q6P3QKrrwaLPSUzEfN3U5XjDQQicW57aC89QxESSUk8mdR+S250Xkj9nLcy6keqfgmY7Sr+m1RXAtvDzTQ1lVkxpKNLUZ/soaGp5tTr4Imvwis/z7paAyrfVayTb35xNMit3IHPUk/dld8ccaJ7tV2daNfMGachsDjVTIJkIm0IfMdL0uNPT1KLMnuWZghi6aqhmiSY3S2pMkq90WrMa62qJS4EoWAPG5efwi8OhXDFEyMH0hSipgXmn63CQxv/E1A5glO8YYTPX7FEMSiPIJGU+IZjJYfEglOkMwSGR6AwmeCiz6l67ZfuAqBB++clAr0j5CW++Y+9/OSZg2w75mNP1xCH+0J0+MIMBKO8b+hf+PbR1tGPabao0tXObSo8ZLaxLegeu0egzy0+2ZPFU413Hiy5GF75BSTi2fcFKqwzpJOaX5xhCP7+X8yRHXyv9j/yxq23tQ/S4nbQVI5ceR48lmrVCBbxpw1Bf2HV0axl28zYLSZ8oVg68ZxQCVJfxIc7mYAa1VVsNglqylHUzYPboS7I/MFONrTWYrYMUxOLjhpGTCQldz97iO8+vo/n7OdD9w7+8uiT/OGldnZ3DrHBqRUBVChRDOnEeH++prICKOXRyc8PgOERpFl8IbReAE98Dda/K+UREOqHunSt764OP3c/d4hrzpzPl9+8dsTT/OtvXuHpfX1IKUcv62teA7v/AtYqknWL6TuaHPsX2+ZS3crJ2MmdIzgROP0G+O210PYPWHGF2pZMavISFZiOl4/6JWlDsOvP8PLd/M39DjbJlXl333bMN+78AGhS1JoCqS/io8rswB4NlGQIQIV7BkLRdGhIMwT+cD/uREI1k3XGqHVax5b7ylyrpkDqD/Uwy2LGYY9QG4wgXS0Ue+aHdnbyuQdUMcgs5vOcA3Y88gu+m1BDFk+ZrRuC8XcV6+g9E/3BKItL/MgEI/EpyQ+A4RGkEQIuvlmViz7/vdQ/0hJO6wxJKfmf+7fjdlj490vzu5FnL6qnNxBhf08w7/1ZtKxVUhRHniPsVXHisiaTZWLXDAEYoaHxsuwydeX/8t3pbeFBZWQnIjQEan5x3z4Y6oQHPgazTuGx2R8YOaUMGArHONAbZO04w0KgSVFrCqS+qA+3RdNZKtEQeJ0qFp7qSZBRkBLfcB+epGYIgpFxN5MBuKtVZZBPk8QwmUN4k3H6zMULOX656QizPA52fP4yHrz5GmKzTudf5+ziiX/fyMP/9lo21vWr3psKentpmZrSPYLAFI2pBMMQZDN3A6y4Ep65g1r8CAG26EDKEPzx5WO8eGiAm16/omDc7+xFat/nD/TlvT8LPWEc9uGrVjHZcSWLdY0ex0ncUHYiYLYqcbC2f4BPM64T0VWcSf0SJVh4z3vVEJW3/BiX05lXr2bHcZUorohH4KglKQTBYDeDkUG8QqvsKcMQ+EKxVJeyXwCJqBKc03SGBoKxcSeKAdyaAJ4/PEBSJokkg7iTSV71FVaLPdwX5Km2Xt55xnyq7RbcDivWNW/C1v0qC0w9LGlyYerdq8JCFezGHyFcWQJqFsHUhIYMQ5DLhf8PYkEsz95OY5XAngiCsx7fcIyv/G0X6+d5efvp8wo+fEG9kxa3ozRDkFEb3mNXzUzjCg3pGB7B+Dnt3WpIyZZfqtsT0VWciV45dPgZuOwWaFyG12nNViDV2H5MGfyKeARVKuw5GOhUyqMSNS/DU/gznokeGrKarVQJiwozRYP4Y4HUUJr+0PjlJdRa1UWWLzJIMBYkicSTSPJMV+Gy1F+9cASzSfCOMzJez8qr1G+9eqhnT0UTxZAZGirSVJZDMBLHaXgEJwhNK+CUd8ELP+Ish9Zt6Kzntof20heMcsvVa4rGOoUQnL2ojucP9KdnHhfC4Ul1sx4xqZrtcYWGUn8bOYJxU7cIFm2El3+m+ksmqqtYR69hX3Y5bHgfkG7Y8oezk9avtvuY7XHQMMZB8Jl4tFCXP9StlEfjMZUwt5R24vY6bam5CG5LlSrvjATwx4dTgnMDFdAZgrQkhj86lKUz9NhxC8PRxIj9I/EE92xu5+KVTbRkKsrWLVQSJrsegOEBZeQrmCgGJVzpslvyS1EXIBCJ4zJyBCcQG28CJB+PqcmZRyNV/Oy5Q1x31oKSyvXKyxMoKYX9yVkIMY42/EyPwKgaqgynXa/kJvY/lqE8OkHJ4oZl8Nb/gzd/PxWi0MOPgznhoe0VShQDePW4e7hP5QiioVFVR7Mer81WllKqUlSTiVC4nzhJPFKQtHsYqJBH4LQ4sUjwxTKUR5NJjsXdeT3wB3d00ReM8q6z8ow7XXmVmlGw/zF1u8IeAZTQXZzDVI2pBMMQ5Mc7D874AEtjewD40Ut+vE4bny6QIM6lrDzB+mvhjA9wPGiiwWXHYh7jv8RuhIYqzoo3qPzQy3ep+L3JWjEJghEIAWvflvX8+pwBXeoZwF/BRDGAx6VKLwfDAypHMOwrOT8ASnI5npQEo4mUBIQ/pIym2+rCF06QlONvJgPlbbsx4U8MpzyCGks1wurg8T3dI/b/1abDzKur4oIleWTFV12tfj/1TfW7gs1kOrVlGAIpJcHo1JWPGoagEBf8G2GT0nF5rlNw0+tX4ClR4KusPMGKK+ANX6d7KDz2RDGkh9OA4RFUCotdiYPt+Ztq/HM1Taq8d6aEg04qPzDXW5FjuKtUj0xHuE8pj0aHyzIE+hoHglE8thp8ZhM+TajRY3en6ugr4REAuIUFXyKS8gi8VQ2cs6iex/f2ZO23rzvA8wf6edeZC/KHchuWqJnlXdvA4kgLDlaQ+jIMQSiaQMqpUR4FwxAUprqBl+ddT0RaaJmzgLedVlx3JZOy8gQa3UORcRqCDAkMm5EjqBinXa86v/c9PHFhoQJkirrpVDJRDOmZBEd1wblEskyPID03QZei9g+rGR5uR32GzlCFDIHJij8ZS+cIqpvZuLyJw30hDvWmQ7G/2nQEq1nw9g1FvrerVB8B9UsnZPZFOaGhqZxOBoYhKErP+o9ycfwObnrLOWU3w5SVJ0A3BOPoEtVDQ9bq1MQjgwrQsDStJDtRieICZJ5kdV5t9zHHW1WxE6vVbKVawpG4EovzJMszBCm57FAUj8OL32TCFxlQz+VsrLgh8Jgd+ImncwTu2Wxcrgy0Hh4KxxL84eV2LlvdUjyhrlcPVThRrKMPtyrlYjCoJbtdRmjoxOONp8zhgc+8ndWzy7/6KidPkEhK+gKRsVcMQdoLMMJClee069XvieoqLkCNw4IQ2TmC7cd84xaay8WDmcNJpRXtScrCk/XyUJsxUtPtqCVsMtETUmEat2sWA8HxjdPMxa1pI/kjPmxS4nDPZUF9Na31zlR46C+vduAbjnFtviRxJk0r4fR/VhPDJoC6ahvReDJ1ki+G7hE4jaqhEw+TSYz5A7yg3sksj4PnSjAEfYEISTmOZjJIewRGorjyrLpK1dU3j5QUmUhMJoGnyprKEfhCMQ71hcYtNJeLR1jpFupk5alqUBLJJZIlRa31JLQHVNm1xz0nVT5ZV4FkMYDH6lLhp6HjeBKJ1ByCjcubeP5AH+FYgl9uOsyixmrOXlRX/MmEgDferjrJJ4BUL0EJ3cVTOZQGDEMwYag8QT2bDvSN6hp2D6mmk8bxhIb08lGjh6DyWKvgE1vh7A9O+qG9VdZUaGj7cRUOqbhHYE5fgHhKbCRL7V+VnqTm1hq+jg53Y5GSqpq5DASjVFnNVI1xrnIubpubIbOJwaHj2ohKpTz62uWNhGNJfvbcIV4+Msi7zpw/5hGelULvLi46qUzDyBFMY85eVEdvIMr+nkDR/br8yi0fX2hISxYboaGJYQKSiaXgcdpSoaFtFU4Up45hTks0eGpLzw8AWM0mauwWBoejuHWPIBbAnUwialoq1lWs49amwx0LHs+aVXzOonrsFhNff3AvNouJt51eenHHRFGnyW6XkjA2cgTTGD1P8NyB/qL76R5Bs7sSHoFhCKYTtU4rPi00tK3dx7y6qlQ4plJ4rOoioiqZxF6/tOzHe6uV16JLUbcTz9AZqqwh8DhUuOdIdDBrVrHDauasRfVEE0muXDur4u/RWNCbQ0vpLjZyBNOY+XUqTzBawrjbr4WGxiMZoM8tNjyCaYW3ypryCF49Nsi6Od6KH8OjFRq4y6wY0klJUWsXIWGT0HSGmugPRiuWKAZwO5XXESKJW8qskt6LVii5jGvPrnxPwFjIlKIejakODRl1hhOInid4qq2n6HyC7qEwtU4rNss47fKp1yl9HINpg9dpYyAYZTAU5Wj/MO86c5RKmDGgX8l7y+whSD2+ypo1rhLAjQls1fSHoixsKGHMa6nHcqZF/9xmR1bI7poz57N6tpvTF4ySJJ4knDYzNospVTlVjKA2uL66QrmUcjE8ggmmlDzBuHsIdN7wdVh55fifx+CEwVNlxR+Os+XoIFD5RDGkwy2eZLIsnSGdWqcNXyhKja0GodVFuE16x3HpoxpLQZ9JAOC2ZhdG2CwmNrSeGEYA1IWg3kswGsFoHLvFNHaJmXFiGIIJppQ8QffQOHsIDKYtep3+s/tVeHHNGHpaRkOXd/aYrNmaVSVSqw2nMQkTNdqsMI+lmkg8QSASH7uQYr611sxJ/20/8edulNpdPJVDacAwBBNOKXmCHn+YxvH0EBhMW/Sk55N7e1hQ7yxZ76ocdCnq3CvsUvE6bfjDMRJJiVuok5nb5mIgqHIblfUIMkJDVXnE5E4w6kr1CCLxKcsPgGEIJpzR+gmSSUlPoEKhIYNph37i3905VPGyUR2vU4VbvI6xhVW8TitSgm84hkczBB57bVpeooIVPDaLnSrte+SZqLGhFUQJz5XSRzB1EtRgGIJJoVieYCAUJZaQ4+sqNpi26MJzUPn+AR2PW9Xce6vHpqVUm9Fd7Dapz7HbWXnBOZ0aqcJPbk1C+0SmrtpeUmexGlw/NYliMAzBpHDOIuXC5ssTVKSHwGDakqnjX6lhNLnU1y/llrlv4Mpz/2tMj08Lz8XwmNXn2FPdXHEJah036oTpdpfXBT0V1LtsBKMJwrHiekNqFoHhEUxr5tVVMdvj4KfPHOTv2zuyZtDqhsBIFhvkw5uRE6i0xlAmb7roVhrGOKUrU2/IbVFdym7X7IoLzul4TOo9KbcLeiootZcgeDIni4UQdUKIh4QQbdrvvOObhBDXa/u0CSGuz3P/A0KI7eNZy4mMEIL/eeNqIrEkH/zFy7z2fx/n+0/sZzAUpVuXlzBCQwZ5qHFYEQIWNlTjdlQ+UVwJUgqkoRgei+oZ8Ljn0heMIkR2eKsS6KWpNXWLK/q8E0HphiAxZdPJYPwNZTcBj0gpbxVC3KTd/s/MHYQQdcDngA2ABF4SQjwgpRzQ7n8LUFyMZxpw+ZoWLl7ZxMO7urn72UPc+rfd3P7wXubVqiloRrLYIB9mk6DWaZuw/EAlSE0pC0XxaN3FHu8iNbWsylrx2niPpQpneBCr8+SoGoLSPIKpkpeA8RuCNwEbtb/vBh4nxxAAlwEPSSn7AYQQDwGXA78WQriAfwNuBH43zrWc8FjMJi5f08Lla1rY3enn7mcPce8rx2h22yumzmgw/fjONacyr8451csoiNthwWwSDIZivPesT1O79afUN66gP/RyRSuGdN6y+gZWHn9uUseGjpVSDIE+r3gqQ0PjPXKzlLJD+7sTyFd2MAc4mnG7XdsG8EXgG0BotAMJIW5EGQzmzz8xtETGw4oWN195yzr+8/IVDI+SSDKY2Zybb/j6CYQQQpOZiFJXv5arLvwyQMUF53ROXXcdp667ruLPOxHo/UE9Q4VLSMOxJMkpnFcMJRgCIcTDQEueu/4784aUUgohigvvZz/vemCxlPKTQojW0faXUv4Q+CHAhg0bSj7OiY7XacM71YswMBgnXqc1a5IaqKvgE9mTmQxq7BacNjMdvnDBfdJDaU7gHIGU8uJC9wkhuoQQs6SUHUKIWUB3nt2OkQ4fAcxFhZDOATYIIQ5p62gSQjwupdyIgYHBSUWt05aapKbTH4xyylzv1CzoBEEIQYvHkZo5ko+plqCG8ZePPgDoVUDXA/fn2edB4FIhRK1WVXQp8KCU8ntSytlSylbgfGCvYQQMDE5OvFXWlKQEqLj3QKiyEtQnKy1uBx2+4YL3B6ZYghrGbwhuBS4RQrQBF2u3EUJsEEL8GEBLEn8ReFH7+YKeODYwMJgeeJ02fBmhoaFInFhCVlRw7mRFeQSFcwSh1HSyEzhHUAwpZR9wUZ7tm4H3Z9z+CfCTIs9zCFgznrUYGBhMHUqBNB0amqhmspORFrcKDSWTEpNpZKVTKjQ0hTkCo7PYwMBg3NRW2whFE0Ti6uo2rTN0YjbBTSazPA7iSUlvAfG5dLL45A0NGRgYGOCpSncXQ6YhMDrmdR2xzgKVQ6HoyZ8jMDAwMMhQIM0xBCfAEPmpZpZH6S8VMgQBbUyl6ySuGjIwMDBI6Q3peQL9d60RGqLZo7yizgIlpEaOwMDAYFrgSQnPKQPQH4xhM5umNO59otBQbcdiEgU9gmAkjs1iwjpF84rBMAQGBgYVoDYlPKeHhiLUVlsRJ4Ee0ERjMgma3Y7ChmCKdYbAMAQGBgYVYGSOIJY1VGem0+JxFAkNTa0ENRiGwMDAoAJU2czYLaZUaGggNDGCcycrLUU8gkAkTvUUJorBMAQGBgYVwpvRVNY/QcqjJyu6RyDlSL3M0BSPqYTxy1CfMMRiMdrb2wmHC4s7Gcw8HA4Hc+fOxWo1qlcmGiU8ly4fNQxBmha3g1A0gT8cT/Vc6AQiiRHbJptpYwja29upqamhtbXVSFAZAEr4rK+vj/b2dhYuXDjVy5n2eJ1WBkMx4okkvmEjR5BJs0c1lXX5wyNO+sFInNmeqZ1QOG1CQ+FwmPr6esMIGKQQQlBfX294iZOEt8rGQCiaqhyqdxmGQGeWdqLPN5cgGJn60NC0MQSAYQQMRmB8JiaP2morA6FYupnM8AhStGgyE10FDIFRPmpgYDAtUFLUUfoCus6QYQh0mtyquzjXI1Dzio3y0WmF2Wxm/fr1qZ9Dhw7x+OOP4/F4UtsuvvhiPvKRj7B+/XpWrVpFVVVV6r577rkn6/nuu+8+du7cWbH1fec732HJkiUIIejt7U1tl1Ly8Y9/nCVLlrBu3TpefvllAPbs2cPpp5/OunXreO655wCIx+NcfPHFhEKjjpk2mGHUOq3EEpL2AfXZMAxBGrvFTH21bUQvQSSeJJGUUzqdDKZRsvhEoKqqii1btmRtO3ToEBdccAF//vOfR+x/6NAhrrzyyhGP0bnvvvu48sorWbVqVUXWd95553HllVeycePGrO1/+9vfaGtro62tjU2bNvGhD32ITZs28YMf/IBvfetbtLa28olPfII//OEPfO973+O6667D6ZzZs2gNRuKtUif+A71BwDAEueQbWXkiSFDDNDUEn//TDnYe91f0OVfNdvO5N66u6HMW49lnn+WBBx7giSee4JZbbuEPf/gDQ0NDfPCDHyQUCrF48WJ+8pOfUFtby8aNGznllFN44okniMfj/OQnP+HMM88c8Zynnnpq3mPdf//9vOc970EIwdlnn83g4CAdHR1YrVZCoRChUAir1crg4CB/+tOf+Pvf/z7RL9/gJMSr6Q3t7w5k3TZQtLgdHM8JDYU05VEjWTyNGB4eToV53vzmN6e2P/XUU6ntX/rSl0p6rnPPPZerrrqK//3f/2XLli0sXryY97znPXz1q1/l1VdfZe3atXz+859P7R8KhdiyZQvf/e53ee9731vWuo8dO8a8efNSt+fOncuxY8f4yEc+wpe//GWuv/56PvOZz/DFL36Rz3zmM5hMxsfGYCT6NLKDvUFcdgt2y9TGvU80insEU/teTUuPYDKv3DPJFxoCCoaGysHn8zE4OMhrX/taAK6//nre/va3p+6/5pprAHjNa16D3+9ncHAQr9c7rmPOnz+fxx9/HIB9+/bR3t7OypUrefe73000GuWLX/wiy5YtG9cxDKYPuhT14b4QLVNcF38i0uJ20B+MEo4lcFjViT+oDaWZ6hyBcWk3TcgtkxRCcNlll7F+/Xre//73F3iUYs6cORw9ejR1u729nTlz5mTt89///d/ccsst3HHHHbz//e/na1/7WpZHYmDg0XIE0UTSmFWcB904dmcMstdnERihIYOC1NTUMDQ0BIDH46G2tpannnoKgJ///Ocp7wDgt7/9LQBPP/00Ho8Hj8fDgw8+yJYtW/jxj39c9DhXXXUVP/vZz5BS8vzzz+PxeJg1a1bq/ieeeILZs2ezdOlSQqEQJpMJk8lkVA4ZZJGZE6gz8gMjaEk1lQ2ntgX16WRGstigEO985zv5wAc+wB133ME999zD3XffnUoWL1q0iJ/+9KepfR0OB6eeeiqxWIyf/OQneZ/vjjvu4Gtf+xqdnZ2sW7eOK664gh//+MdcccUV/PWvf2XJkiU4nc6s55VScsstt6QMzY033si1115LPB7ne9/73sS+AQYnFVaziRq7haFI3JhVnAe9uzizhDQ1ncxm5AimDYFAYMS2jRs3jijX1GltbWX79u0Fn++8884b0Ufw/PPP5933uuuu4/bbby+6vo9//ON8/OMfH7FdCMGdd96Z9zFCCB566KHU7ZUrV6b6DAwMcvE4rZohMDyCXPINsT9RykeN0JCBgUHF0GUljBzBSGocVlx2S5ZHEIqeGDkCwyOYBuiVPQYGU42eJ6g3DEFemt32HI8ggc1swmaZ2mtywyMwMDCoGCmPwBCcy0vuyMpgJI5zinsIwDAEBgYGFUT3CAx5ify0uKuyPIJgdOrHVIJhCAwMDCqI18gRFKXFY6d7KEIiqUZWnggS1GAYAgMDgwqyoM5JldVMU41RPpqPFk8ViaSkN6CayoKRqZegBsMQVJT29nbe9KY3sXTpUhYvXswnPvEJotFo3n2PHz/O2972tlGf84orrmBwcHBM67n55pv5+te/nnf7nDlzWL9+PWvWrOGBBx4Y0/NXmi9/+ctTvQSDcXL1qXN44j82UuMwykfz0ZJTQho4AaaTgWEIKoaUkre85S1cffXVtLW1sXfvXgKBAP/93/89Yt94PM7s2bNHzB/Ix1//+tdxawbl45Of/CRbtmzh97//Pe9973tJJpMlPS6RSFR8LTpjMQQTuR6D8jGbBE01hs5QIXJHVoZOkBzB1K9gIvjbTdC5rbLP2bIWXn9rwbsfffRRHA4H//zP/wyoITW33XYbCxcu5POf/zy/+93v+OMf/0ggECCRSHD33Xdz5ZVXsn37dkKhEDfccAPbt29n+fLlHD9+nDvvvJMNGzbQ2trK5s2bCQQCvP71r+f888/n2WefZc6cOdx///1UVVXxox/9iB/+8IdEo1GWLFnCz3/+85LnBaxcuRKLxUJvby833ngjR48eJRwO84lPfIIbb7wRAJfLxb/8y7/w8MMPc+edd/Loo4/ypz/9ieHhYc4991x+8IMfIIRg48aNnHrqqTz11FMEg0F+9rOf8ZWvfIVt27bxjne8g1tuuQWAX/ziF9xxxx1Eo1HOOussvvvd7/Lf//3fKfXW1atX88tf/jLvfmazecR6/vznP/PAAw9gsVi49NJL83pBBgYnAnpTma5CqkJDU38aNjyCCrFjxw5OP/30rG1ut5v58+ezb98+AF5++WXuuecennjiiaz9vvvd71JbW8vOnTv54he/yEsvvZT3GG1tbXzkIx9hx44deL1e/vCHPwDwlre8hRdffJGtW7eycuVK/u///q/kdW/atAmTyURjYyM/+clPeOmll9i8eTN33HEHfX19AASDQc466yy2bt3K+eefz0c/+lFefPFFtm/fzvDwcJayqs1mY/PmzXzwgx/kTW96E3feeSfbt2/nrrvuoq+vj127dvHb3/6WZ555hi1btmA2m/nlL3/JrbfemlJv/eUvf1lwv9z1rFy5knvvvZcdO3bw6quv8tnPfrbk125gMNnUV9uwmkXKIwhE4lMuQQ3T1SMocuU+lVxyySXU1dWN2P7000/ziU98AoA1a9awbt26vI9fuHAh69evB+D000/n0KFDAGzfvp3PfvazDA4OEggEuOyyy0Zdy2233cYvfvELampq+O1vf4sQgjvuuIN7770XgKNHj9LW1kZ9fT1ms5m3vvWtqcc+9thjfO1rXyMUCtHf38/q1at54xvfCCgBO4C1a9eyevXqlHjdokWLOHr0KE8//TQvvfQSZ5xxBqBmODQ1NY1Y3yOPPFJwv8z1eDweHA4H73vf+7jyyiu58sorR33tBgZThUkLnekeQSgax3kCeARTv4JpwqpVq0bE/P1+P0eOHGHJkiW8/PLLVFdXj+sYdnu6EsNsNjM8rFQMb7jhBu677z5OOeUU7rrrrpI6jT/5yU/y6U9/OnX78ccf5+GHH+a5557D6XSyceNGwmH1YXU4HJjN6qolHA7z4Q9/mM2bNzNv3jxuvvnm1H6ZazSZTFnrNZlMxONxpJRcf/31fOUrXym6vmL7Za7HYrHwwgsv8Mgjj3DPPffwne98h0cffXTU129gMFXM8jjo8A0TiSeIJaRRPjqduOiiiwiFQvzsZz8DVBLzU5/6FDfccMOo8frzzjuP3/3udwDs3LmTbdvKy28MDQ0xa9YsYrFYKnxSLj6fj9raWpxOJ7t37y4obqef9BsaGggEAiUlvDO56KKLuOeee+ju7gagv7+fw4cPA2C1WonFYqPul0kgEMDn83HFFVdw2223sXXr1rLWY2Aw2TR7HHT5IykJ6uopVh6FcRoCIUSdEOIhIUSb9ru2wH7Xa/u0CSGuz9huE0L8UAixVwixWwjx1nyPPxkQQnDvvffy+9//nqVLl7Js2TIcDkdJlTAf/vCH6enpYdWqVXz2s59l9erVeDyeko/9xS9+kbPOOovzzjuPFStWjGn9l19+OfF4nJUrV3LTTTdx9tln593P6/XygQ98gDVr1nDZZZelQjelsmrVKm655RYuvfRS1q1bxyWXXEJHRwegJK7XrVvHtddeW3S/TIaGhrjyyitZt24d559/Pt/85jfLf/EGBpPILLfyCFIS1CeARyCklGN/sBBfA/qllLcKIW4CaqWU/5mzTx2wGdgASOAl4HQp5YAQ4vOAWUr5WSGECaiTUvaOdtwNGzbIzZs3Z23btWsXK1euHPNrmUoSiQSxWAyHw8H+/fu5+OKL2bNnDzab0Z1ZCU7mz4bB9OPHTx3glr/s4rc3ns07fvg83732NK5YO2v0B1YAIcRLUsoNudvHa4reBGzU/r4beBz4z5x9LgMeklL2awt5CLgc+DXwXmAFgJQyCYxqBKYjoVCI173udcRiMaSUfPe73zWMgIHBNEWfVLa/JwhMvQQ1jN8QNEspdX+9E2jOs88c4GjG7XZgjhDCq93+ohBiI7Af+KiUsmucazrpqKmpIdfDMTAwmJ7o3cX7e9QgqxOhfHTUHIEQ4mEhxPY8P2/K3E+qGFM5cSYLMBd4Vkp5GvAcULATSAhxoxBisxBic09PTxmHMTAwMDhx0JvK9nUrQ+A8GTqLpZQXF7pPCNElhJglpewQQswCuvPsdox0+AjUyf9xoA8IAX/Utv8eeF+RdfwQ+CGoHMFo6zYwMDA4EWke4RFMvSEYb/noA4BeBXQ9cH+efR4ELhVC1GpVRZcCD2oexJ9IG4mLgJ15Hm9gYGAwbbBZTDS4bBwbVH1AJ0KOYLyG4FbgEiFEG3CxdhshxAYhxI8BtCTxF4EXtZ8v6IljVGL5ZiHEq8C7gU+Ncz0GBgYGJzwtHgd6weZJL0MtpeyTUl4kpVwqpbxYP8FLKTdLKd+fsd9PpJRLtJ+fZmw/LKV8jZRynfY8R8aznqnGbDanpJ3f/va3EwqFpnpJPP744zz77LPjeo5Dhw7xq1/9qkIrgt27d3POOedgt9tHCMT9/e9/Z/ny5SxZsoRbb01LhVx77bWsW7eOz3zmM6ltt9xyC/fdd1/F1mVgMFnoCWOLSWAzT31f79SvYBqhi6Zt374dm83G97///ZIeF4/HJ2xNYzEEueuptCGoq6vjjjvuyJK4ANVP8ZGPfIS//e1v7Ny5k1//+tfs3LmTV199laqqKl599VVefPFFfD4fHR0dbNq0iauvvrpi6zIwmCz0EtJquwUhxBSvZppqDX31ha+yu393RZ9zRd0K/vPM3BaJwlxwwQW8+uqr/OlPf+KWW24hGo1SX1/PL3/5S5qbm7n55pvZv38/Bw4cYP78+XzlK1/h3e9+N8Ggqi3+zne+w7nnnsvjjz/O5z73ObxeL9u2beOf/umfWLt2Ld/61rcYHh7mvvvuY/HixfT09PDBD36QI0eUU3X77bczZ84cvv/972M2m/nFL37Bt7/9bVasWDFiv/POO2/Een7961+nXstNN93Erl27WL9+Pddffz0f+tCH+NCHPsTmzZuxWCx885vf5HWvex133XUX9957Lz6fj2PHjnHdddfxuc99bsR709TURFNTE3/5y1+ytr/wwgssWbKERYsWAfDOd76T+++/n6uvvprh4WGSySSxWAyz2cz//M//8PnPf768f6KBwQmC7hGcCIlimKaGYKqJx+P87W9/4/LLL+f888/n+eefRwjBj3/8Y772ta/xjW98A1C6Qk8//TRVVVWEQiEeeughHA4HbW1tXHPNNanegq1bt7Jr1y7q6upYtGgR73//+3nhhRf41re+xbe//W1uv/12PvGJT/DJT36S888/nyNHjnDZZZexa9cuPvjBD+JyuVJX3+9617vy7pe7nkxuvfVWvv71r6fkpr/xjW8ghGDbtm3s3r2bSy+9lL179wLqZL59+3acTidnnHEGb3jDG9iwYUQjY16OHTvGvHnzUrfnzp3Lpk2bWLlyJY2NjZx22mm8+93vZt++fSSTSU477bRx/JcMDKaOFo/6jjlPAJ0hmKaGoJwr90qiD1YB5RG8733vY8+ePbzjHe+go6ODaDTKwoULU/tfddVVqZNuLBbjox/9aEp7Xz+xApxxxhkpOefFixdz6aWXAkrq+bHHHgPg4YcfZufOdNGV3+8nEAiMWGOx/TLXU4ynn36aj33sYwCsWLGCBQsWpNZ7ySWXUF9fD6g5CU8//XTJhqAYt99+e+rvN77xjfzgBz/gS1/6Elu3buWSSy7hAx/4wLiPYWAwWegewYlQMQTT1BBMFXqOIJOPfexj/Nu//RtXXXUVjz/+ODfffHPqvkxZ6ttuu43m5ma2bt1KMpnE4UiP+8uVc86Uetbj+clkkueffz7rcfkott94ZbKBEfFOIQR33nknP/rRjwA1enP27Nl5HztnzhyOHk03obe3tzNnzpysfe6//35OP/10AoEA+/fv53e/+x2XXXYZ1157bclT2QwMpho9R3CihIaMZPEE4/P5Uiezu+++u+h+s2bNwmQy8fOf/7zsWbyXXnop3/72t1O3dYNUU1PD0NDQqPsVI/c5LrjggpTc9d69ezly5AjLly8H4KGHHqK/vz+VvzjvvPP4yEc+wpYtW9iyZUtBIwDK82lra+PgwYNEo1F+85vfpAbdgPKabr/9dv7jP/6D4eHhlNFJJBJEo9FRX4eBwYlCOll8YoSGDEMwwdx88828/e1v5/TTT6ehoaHgfh/+8Ie5++67OeWUU9i9e3fZV+d33HEHmzdvZt26daxatSpVsfTGN76Re++9l/Xr1/PUU08V3K8Y69atw2w2c8opp3Dbbbfx4Q9/mGQyydq1a3nHO97BXXfdlfJSzjzzTN761reybt063vrWt+YNC3V2djJ37ly++c1vcssttzB37lz8fj8Wi4XvfOc7XHbZZaxcuZJ/+qd/YvXq1anH3XnnnVx//fU4nU7WrVtHKBRi7dq1nH766Xi93rLeLwODqcRlt1Bjt5wQg+thnDLUU8V0k6GeLtx1111s3ryZ73znO1O9lCyMz4bBicivNh1habOLM1pHjq+dKCZKhtrAwMDAYAy866z5U72EFIYhMKgYN9xwAzfccMNUL8PAwKBMplWO4GQMcxlMLMZnwsBgdKaNIXA4HPT19RlffIMUUkr6+vpGLak1MJjpTJvQ0Ny5c2lvb8cYWmOQicPhYO7cuVO9DAODE5ppYwisVmtW166BgYGBQWlMm9CQgYGBgcHYMAyBgYGBwQzHMAQGBgYGM5yTsrNYCNEDHB7jwxuA3gou52TBeN0zC+N1zyxKfd0LpJSNuRtPSkMwHoQQm/O1WE93jNc9szBe98xivK/bCA0ZGBgYzHAMQ2BgYGAww5mJhuCHU72AKcJ43TML43XPLMb1umdcjsDAwMDAIJuZ6BEYGBgYGGRgGAIDAwODGc6MMQRCiMuFEHuEEPuEEDdN9XomEiHET4QQ3UKI7Rnb6oQQDwkh2rTftVO5xolACDFPCPGYEGKnEGKHEOIT2vZp/dqFEA4hxAtCiK3a6/68tn2hEGKT9pn/rRDCNtVrnQiEEGYhxCtCiD9rt6f96xZCHBJCbBNCbBFCbNa2jflzPiMMgRDCDNwJvB5YBVwjhFg1tauaUO4CLs/ZdhPwiJRyKfCIdnu6EQc+JaVcBZwNfET7P0/31x4BLpRSngKsBy4XQpwNfBW4TUq5BBgA3jd1S5xQPgHsyrg9U17366SU6zP6B8b8OZ8RhgA4E9gnpTwgpYwCvwHeNMVrmjCklE8C/Tmb3wTcrf19N3D1ZK5pMpBSdkgpX9b+HkKdHOYwzV+7VAS0m1btRwIXAvdo26fd6wYQQswF3gD8WLstmAGvuwBj/pzPFEMwBziacbtd2zaTaJZSdmh/dwLNU7mYiUYI0QqcCmxiBrx2LTyyBegGHgL2A4NSyri2y3T9zN8O/AeQ1G7XMzNetwT+IYR4SQhxo7ZtzJ/zaTOPwKB0pJRSCDFt64aFEC7gD8C/Sin96iJRMV1fu5QyAawXQniBe4EVU7uiiUcIcSXQLaV8SQixcYqXM9mcL6U8JoRoAh4SQuzOvLPcz/lM8QiOAfMybs/Vts0kuoQQswC0391TvJ4JQQhhRRmBX0op/6htnhGvHUBKOQg8BpwDeIUQ+sXedPzMnwdcJYQ4hAr3Xgh8i+n/upFSHtN+d6MM/5mM43M+UwzBi8BSrZrABrwTeGCK1zTZPABcr/19PXD/FK5lQtDiw/8H7JJSfjPjrmn92oUQjZongBCiCrgElR95DHibttu0e91Syv+SUs6VUraivtOPSimvZZq/biFEtRCiRv8buBTYzjg+5zOms1gIcQUqnmgGfiKl/NLUrmjiEEL8GtiIkqbtAj4H3Af8DpiPkvD+JyllbkL5pEYIcT7wFLCNdMz4M6g8wbR97UKIdajkoBl1cfc7KeUXhBCLUFfKdcArwHVSysjUrXTi0EJDn5ZSXjndX7f2+u7VblqAX0kpvySEqGeMn/MZYwgMDAwMDPIzU0JDBgYGBgYFMAyBgYGBwQzHMAQGBgYGMxzDEBgYGBjMcAxDYGBgYHCCIoR4uyYkmBRClDSTWAiR0MTotgghSiqTNwyBwQlNxod6uxDi90II5wmwpo1CiHPH+RyzhRD3jL5n1mNuEEJ8R/v7g0KI94xnDQYnFtrn6q6czduBtwBPlvFUw5oY3Xop5VWlPMAwBAYnOvqHeg0QBT5YyoMyOksngo1AWYYgdz1SyuNSyrcV2n80pJTfl1L+bKyPNzg5kFLuklLuyd2uaUv9rxDiRSHEq0KIfxnPcQxDYHAy8RSwRAjxRk1v/hUhxMNCiGYAIcTNQoifCyGeAX4uhGgVQjwlhHhZ+zlX22+jEOIJIcT9QogDQohbhRDXCqXpv00IsVjbr1EI8Qfty/aiEOI8Tczug8AnNU/lgnz75VtP5gvR1rZd+/sGIcQfhRB/F0pL/msZ+/2zEGKvEOIFlKQCGc/9ae3vJdr7sFV7nfr6/z3jRKHPKKgWQvxF23e7EOIdE/KfMpho3gf4pJRnAGcAHxBCLNTucwghNgshnhdCXF3KkxmicwYnBdoV9euBvwNPA2drwlrvR6lPfkrbdRVKkGtYCyNdIqUMCyGWAr8G9DjrKcBKlFz3AeDHUsozhRpm8zHgX1G6NbdJKZ8WQswHHpRSrhRCfB8ISCm/rq3tV7n7ac+dtZ5RXuJ6lFpqBNgjhPg2ar7C54HTAR9KOuGVPI/9JXCrlPJeIYQDMAkhLgWWojRoBPCAEOI1QCNwXEr5Bm3tnlHWZTDBCCE2AXbABdQJpSIL8J9SygcLPOxSYJ0QQvcqPaj/90FggSZItwh4VAixTUq5v9gaDENgcKJTlfHFeAqlJbQc+K1Qwlo21Idf54GMk64V+I4QYj2QAJZl7PeiLtkrhNgP/EPbvg14nfb3xcAqkVYvdQulbJpLsf0eKMEIgBoo4tPWsxNYgJIIeVxK2aNt/23Oa0AozZk5Usp7AaSUYW37paiThW44XKgTxVPAN4QQXwX+LKV8qoS1GUwgUsqzICWTcYOU8oYSHiaAj+UzFBmCdAeEEI+jLjAMQ2BwUjMspVyfuUG7Wv6mlPIB7ctzc8bdwYy/P4nSWjoFFQYNZ9yXqT2TzLidJP29MKE8j8zHkXHCp4T9grk7FyBzPQnG/90UwFeklD8YcYcQpwFXALcIIR6RUn5hnMcymHweBD4khHhUShkTQixDqazagJCUMiKEaECFE79W7InAyBEYnJx4SEsLXz/Kfh1SyiTwbpQoWzn8AxUmAkDzLACGgJoS9hsvm4DXCiHqhZLXfnvuDtoktnY9FiyEsGshsQeB9+qeiRBijhCiSQgxG3Wi+AXwv8BpFVqrwQQghHizEKIdJSv+FyGE7gH8GNgJvKzlmn6AunhYCWwWQmxFhRJvlVLuHO04hkdgcDJyM/B7IcQA8CiwsMB+3wX+IFSZ5d8p/epc5+PAnUKIV1HflSdRieI/AfcIId6EMgCF9hsXUsoOIcTNwHPAILClwK7vBn4ghPgCEAPeLqX8hxBiJfCc5pkEgOuAJcD/CiGS2r4fGu86DSqDlPJx4PGcbfeSVhrN3J5EKet+JueuZ4G15R7bUB81MDAwmOEYoSEDAwODGY5hCAwMDAxmOIYhMDAwMJjhGIbAwMDAYIZjGAIDAwODGY5hCAwMDAxmOIYhMDAwMJjh/H97Wk8QMeVwwQAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(np.arange(100000,100050,1), reverse_top10fft[100000:100050], label = \"FFT top-10%\")\n",
+ "plt.plot(np.arange(100000,100050,1), conc2[100000:100050], label = \"Original Parameters\")\n",
+ "plt.plot(np.arange(100000,100050,1), top10_og[100000:100050], label = \"Parameter top-10%\")\n",
+ "plt.title('Parameter Values') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.legend()\n",
+ "plt.draw()\n",
+ "plt.savefig(\"Parameters.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 83,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAjAElEQVR4nO3de5gddZXu8e9LYiAgkARixCTQYYh4kKMIPRCPo0aCIQE1eEQHRiUy0egAjjrOo8HL4HA5E+Y4InjByUgk4VFCwGGIXAwxwoiXYDoQgYBMmgimM4G0JAGU2wms80ethmKzu3vvTvfu7vT7eZ79dNWqX1Wtqu7ea9evalcpIjAzs6Ftt/5OwMzM+p+LgZmZuRiYmZmLgZmZ4WJgZma4GJiZGS4GZjtN0lRJbf2dRyVJ35H05f7OwwYHF4MhTtKDkp6S9EdJj0i6XNIr+zuvrkhqkhSShvfS8vaQtF3SsVWmXSTpmt5YT2/K39P5FbGX7JeI+EREnFfDsh6UdFxf5WqDg4uBAbw7Il4JHAk0A1+qZ2YVBs3fUmURiYingauA0yraDQNOBRY1LrtdS28VbOt7g+Yf2PpeRGwCbgIOlzRa0vWS2iVty+EJHW0l3SrpAkm/AJ4EDpZ0uqT7JD0haYOkj5faT5XUJulzkrZI2izpJEknSPovSVslfaHUfjdJ8yQ9IOlRSUsljcnJP8uf2/OI5s05z1/n+rdJWi7poNLyQtKZktYD66ts/iLgfZL2LMWOp/gfuamrbauU6zqkNP6ST/GS3iVpbR6N/FLSG0rTPi9pU67nfknTOltPd8rrlbR//g63576+LffxFcCBwI9yX34u279H0rpsf6uk/1Fa7pGS7swcr5Z0VWk9Hb/nz0t6GPhejX9L5+e++KOkH0naT9L3JT0uabWkpp7uB6uNi4G9QNJE4ATgToq/je8BB1G8WTwFfLNilg8Dc4G9gYeALcC7gH2A04GLJB1Zav9qYA9gPPAPwL8BHwKOAt4KfFnSpGz7SeAk4O3Aa4BtwLdy2tvy56iIeGVE/ErSLOALwP8GxgK3AVdW5HsScAxwWOW2R8Qvgc05f3n7fhARO2rYtppIehOwEPg4sB/wr8AySbtLOhQ4C/jziNibohg9WO86OvFZoI1i34yj2FcRER8Gfk8eHUbEP0t6LcW++3S2v5GiWIyQNAK4FrgcGJPt3luxrlfntIMo/j5q+Vs6hWJ/jwf+DPhVzjMGuA84pzd2gnUhIvwawi+KN5s/Atsp3tC/DYys0u4IYFtp/Fbg3G6W/R/Ap3J4KsWbwLAc3xsI4JhS+zXASTl8HzCtNO0A4P8Bw4GmnHd4afpNwJzS+G4URywH5XgAx3aT75eAm3N4n5z/TTVuW1tpWgCHlMYvB87P4UuB8yqWdT9F0TuEougcB7yim1wvB57O31vH6/HyfqlY77nAdeW8Kv4GjiuNfxlYWrEvN+V2vi2HVZr+89J6pgLPAnt0kXu1v6Uvlsb/BbipNP5uYG1//6/s6i8fGRgUb8CjIuKgiDgjIp6StKekf5X0kKTHKbpmRmU/eoeN5YVImilpVXZDbKc4yti/1OTRiHguh5/Kn4+Upj8FdJy8Pgi4NrsptlMUh+coPtVWcxBwcan9VkAUnzSr5lvFFcA7JL0GOBl4ICLurHHbanUQ8NmOPHNZE4HXREQrxafxrwBbJC3JXDrz1fy9jYqIUcAbumj7f4FW4Obs5prXRdvXUHwwACAinqfYd+Nz2qbId+lUuV/bozgPA0CNf0uVfwed/V1YH3ExsM58FjiU4pP7PrzYNaNSmxfeECTtDvwQ+CowLt+cbqxoX4+NwMzym11E7BHFeY1qt9rdCHy8ov3IKLp/XpZvNRHxEEX30ocouiwW9XDbngTK5x5eXZHnBRV57hkRV2YOP4iIv6AoGgFc2FXOtYqIJyLisxFxMPAe4O9K5yMq98t/5/qB4gIBioK1iaIrbXzGOkysXF3FeC1/S9bPXAysM3tTfCLbniduu+uzHQHsDrQDOyTNBKbvxPq/A1zQcRJY0tg8L0Cu43ng4Ir2Z0t6fbbfV9L7e7DeRRT99m8Bvp+xerdtLfBXkoZJmkHRBdTh34BPSDpGhb0knShpb0mHSjo2i8/TFPv/+R5sw8vkSetD8k38MYqjrI5lP8JL9+VS4ERJ0yS9guLN/BnglxR9+c8BZ0kanr+To7tZfb1/S9YPXAysM18HRgJ/AFYBP+6qcUQ8AfwtxRvJNuCvgGU7sf6Lc/6bJT2RORyT63oSuAD4RXa1TImIayk+RS/Jroh7gJk9WO8PKU5aroyIzT3ctk9R9HNvBz5IcX6BXFYL8DGKE6jbKLpuPpKTdwfmU+zzh4FXAWf3YBuqmQz8hOL80K+Ab0fELTntn4Av5b78+4i4n+Lo6BuZy7spTjA/GxHPUpxkn5Pb9yHgeopi0ZmvU8ffkvUPvbTrz8ysPpJuB74TEd/r71ys53xkYGZ1kfR2Sa/ObqLZFCeu/Wl/kPO3A82sXodSdJntBWwATu7oUrPBy91EZmbmbiIzMxvE3UT7779/NDU19XcaZmaDxpo1a/4QEWOrTRu0xaCpqYmWlpb+TsPMbNCQ9FBn09xNZGZmLgZmZuZiYGZmuBiYmRkuBmZmhouBmZlRYzGQ9BkVz0O9R9KVkvaQNEnS7ZJaVTwDdUS23T3HW3N6U2k5Z2f8fknHl+IzMtbazUM3zMysD3RbDCSNp7h9b3NEHA4Mo3he6YXARRFxCMWteOfkLHMoHml3CHBRtkPSYTnf64EZwLfzfu/DKJ5tO5Pi2bSnZlszM2uQWruJhgMjJQ2neILTZuBY4JqcvojiYeMAs3KcnD4tH6gxC1gSEc9ExO8o7uN+dL5aI2JD3it9SbY1M7MG6fYbyBGxSdJXgd9TPK3oZooHl2+PiB3ZrI0XnzU7nnwmakTskPQYsF/GV5UWXZ5nY0X8mGq5SJoLzAU48MADu0vdrFc0zbvhheEH55/YbdxsMOq2GEgaTfFJfRLFk42upujmabiIWAAsAGhubvbtVq3PlN/oa4mbDXa13JvoOOB3EdEOIOnfKZ4PO0rS8Dw6mEDxsGzy50SgLbuV9gUeLcU7lOfpLG42KHRWJHzEYINFLcXg98AUSXtSdBNNA1qAW4CTKfr4ZwPXZftlOf6rnP7TiAhJy4AfSPoa8BqKZ7L+GhAwWdIkiiJwCsUzZs0aqi8+9bsryQaLWs4Z3C7pGuAOYAdwJ0VXzQ0UDx8/P2OX5SyXAVdIagW2Ury5ExHrJC0F7s3lnBkRzwFIOgtYTnGl0sKIWNd7m2jWOXf7mBUG7ZPOmpubw7ewtp3VX8XARwnWHyStiYjmatP8DWQzMxu8D7cxG8x8LsEGGh8ZmJmZi4GZmbmbyKzfucvIBgIXA7MBxIXB+ou7iczMzMXAzMzcTWRDkL91bPZyPjIwMzMXAzMzczEwMzN8zsBswPJlptZIPjIwMzMXAzMzczEwMzNqKAaSDpW0tvR6XNKnJY2RtELS+vw5OttL0iWSWiXdJenI0rJmZ/v1kmaX4kdJujvnuUSS+mZzzcysmm6LQUTcHxFHRMQRwFHAk8C1wDxgZURMBlbmOMBMiucbTwbmApcCSBoDnAMcAxwNnNNRQLLNx0rzzeiNjTMzs9rU2000DXggIh4CZgGLMr4IOCmHZwGLo7AKGCXpAOB4YEVEbI2IbcAKYEZO2yciVkXxDM7FpWWZmVkD1FsMTgGuzOFxEbE5hx8GxuXweGBjaZ62jHUVb6sSNzOzBqm5GEgaAbwHuLpyWn6ij17Mq7Mc5kpqkdTS3t7e16szMxsy6vnS2Uzgjoh4JMcfkXRARGzOrp4tGd8ETCzNNyFjm4CpFfFbMz6hSvuXiYgFwAKA5ubmPi8+tuvwzenMulZPN9GpvNhFBLAM6LgiaDZwXSl+Wl5VNAV4LLuTlgPTJY3OE8fTgeU57XFJU/IqotNKyzIzswao6chA0l7AO4GPl8LzgaWS5gAPAR/I+I3ACUArxZVHpwNExFZJ5wGrs925EbE1h88ALgdGAjfly8zMGqSmYhARfwL2q4g9SnF1UWXbAM7sZDkLgYVV4i3A4bXkYmZmvc/fQDYzMxcDMzNzMTAzM1wMzMwMFwMzM8PFwMzMcDEwMzNcDMzMDBcDMzPDxcDMzHAxMDMzXAzMzAwXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM6PGYiBplKRrJP1W0n2S3ixpjKQVktbnz9HZVpIukdQq6S5JR5aWMzvbr5c0uxQ/StLdOc8l+SxkMzNrkFqPDC4GfhwRrwPeCNwHzANWRsRkYGWOA8wEJudrLnApgKQxwDnAMcDRwDkdBSTbfKw034yd2ywzM6tHt8VA0r7A24DLACLi2YjYDswCFmWzRcBJOTwLWByFVcAoSQcAxwMrImJrRGwDVgAzcto+EbEqn5+8uLQsMzNrgFqODCYB7cD3JN0p6buS9gLGRcTmbPMwMC6HxwMbS/O3ZayreFuV+MtImiupRVJLe3t7DambmVktaikGw4EjgUsj4k3An3ixSwiA/EQfvZ/eS0XEgohojojmsWPH9vXqzMyGjFqKQRvQFhG35/g1FMXhkeziIX9uyembgIml+SdkrKv4hCpxMzNrkG6LQUQ8DGyUdGiGpgH3AsuAjiuCZgPX5fAy4LS8qmgK8Fh2Jy0HpksanSeOpwPLc9rjkqbkVUSnlZZlZmYNMLzGdp8Evi9pBLABOJ2ikCyVNAd4CPhAtr0ROAFoBZ7MtkTEVknnAauz3bkRsTWHzwAuB0YCN+XLzMwapKZiEBFrgeYqk6ZVaRvAmZ0sZyGwsEq8BTi8llzMzKz31XpkYGb9qGneDS8MPzj/xH7MxHZVvh2FmZm5GJiZmYuBmZnhYmBmZrgYmJkZLgZmZoaLgZmZ4WJgZma4GJiZGS4GZmaGi4GZmeFiYGZmuBiYmRkuBmZmhouBmZnhYmBmZtRYDCQ9KOluSWsltWRsjKQVktbnz9EZl6RLJLVKukvSkaXlzM726yXNLsWPyuW35rzq7Q01M7PO1XNk8I6IOCIiOh5/OQ9YGRGTgZU5DjATmJyvucClUBQP4BzgGOBo4JyOApJtPlaab0aPt8jMzOq2M91Es4BFObwIOKkUXxyFVcAoSQcAxwMrImJrRGwDVgAzcto+EbEqn5+8uLQsMzNrgFqLQQA3S1ojaW7GxkXE5hx+GBiXw+OBjaV52zLWVbytSvxlJM2V1CKppb29vcbUzcysO8NrbPcXEbFJ0quAFZJ+W54YESEpej+9l4qIBcACgObm5j5fn5nZUFHTkUFEbMqfW4BrKfr8H8kuHvLnlmy+CZhYmn1CxrqKT6gSNzOzBum2GEjaS9LeHcPAdOAeYBnQcUXQbOC6HF4GnJZXFU0BHsvupOXAdEmj88TxdGB5Tntc0pS8iui00rLMzKwBaukmGgdcm1d7Dgd+EBE/lrQaWCppDvAQ8IFsfyNwAtAKPAmcDhARWyWdB6zOdudGxNYcPgO4HBgJ3JQvMzNrkG6LQURsAN5YJf4oMK1KPIAzO1nWQmBhlXgLcHgN+ZqZWR/wN5DNzMzFwMzMXAzMzAwXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM8PFwMzMqP2upWY2QDTNu+GF4Qfnn9iPmdiuxEcGZmbmYmBmZi4GZmaGi4GZmeFiYGZmuBiYmRkuBmZmRh3FQNIwSXdKuj7HJ0m6XVKrpKskjcj47jnemtObSss4O+P3Szq+FJ+RsVZJ83px+8zMrAb1HBl8CrivNH4hcFFEHAJsA+ZkfA6wLeMXZTskHQacArwemAF8OwvMMOBbwEzgMODUbGtmZg1SUzGQNAE4Efhujgs4FrgmmywCTsrhWTlOTp+W7WcBSyLimYj4HdAKHJ2v1ojYEBHPAkuyrZmZNUitRwZfBz4HPJ/j+wHbI2JHjrcB43N4PLARIKc/lu1fiFfM01n8ZSTNldQiqaW9vb3G1M3MrDvdFgNJ7wK2RMSaBuTTpYhYEBHNEdE8duzY/k7HzGyXUcuN6t4CvEfSCcAewD7AxcAoScPz0/8EYFO23wRMBNokDQf2BR4txTuU5+ksbmZmDdDtkUFEnB0REyKiieIE8E8j4oPALcDJ2Ww2cF0OL8txcvpPIyIyfkpebTQJmAz8GlgNTM6rk0bkOpb1ytaZmVlNduYW1p8Hlkg6H7gTuCzjlwFXSGoFtlK8uRMR6yQtBe4FdgBnRsRzAJLOApYDw4CFEbFuJ/IyA156q2cz61pdxSAibgVuzeENFFcCVbZ5Gnh/J/NfAFxQJX4jcGM9uZiZWe/xN5DNzMzFwMzMXAzMzAwXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM8PFwMzMcDEwMzNcDMzMDBcDMzPDxcDMzHAxMDMzXAzMzAwXAzMzw8XAzMyooRhI2kPSryX9RtI6Sf+Y8UmSbpfUKumqfH4x+YzjqzJ+u6Sm0rLOzvj9ko4vxWdkrFXSvD7YTjMz60ItRwbPAMdGxBuBI4AZkqYAFwIXRcQhwDZgTrafA2zL+EXZDkmHUTwP+fXADODbkoZJGgZ8C5gJHAacmm3NzKxBui0GUfhjjr4iXwEcC1yT8UXASTk8K8fJ6dMkKeNLIuKZiPgd0ErxDOWjgdaI2BARzwJLsq2ZmTVITecM8hP8WmALsAJ4ANgeETuySRswPofHAxsBcvpjwH7leMU8ncWr5TFXUouklvb29lpSNzOzGtRUDCLiuYg4AphA8Un+dX2ZVBd5LIiI5ohoHjt2bH+kYGa2S6rraqKI2A7cArwZGCVpeE6aAGzK4U3ARICcvi/waDleMU9ncTMza5BariYaK2lUDo8E3gncR1EUTs5ms4HrcnhZjpPTfxoRkfFT8mqjScBk4NfAamByXp00guIk87Je2DYzM6vR8O6bcACwKK/62Q1YGhHXS7oXWCLpfOBO4LJsfxlwhaRWYCvFmzsRsU7SUuBeYAdwZkQ8ByDpLGA5MAxYGBHrem0LzcysW90Wg4i4C3hTlfgGivMHlfGngfd3sqwLgAuqxG8EbqwhXzMz6wP+BrKZmbkYmJmZi4GZmeFiYGZmuBiYmRkuBmZmhouBmZnhYmBmZtT2DWSzQaFp3g39nYLZoOUjAzMzczEwMzMXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM8PFwMzMqO0ZyBMl3SLpXknrJH0q42MkrZC0Pn+OzrgkXSKpVdJdko4sLWt2tl8vaXYpfpSku3OeSySpLzbWzMyqq+XIYAfw2Yg4DJgCnCnpMGAesDIiJgMrcxxgJsXD7icDc4FLoSgewDnAMRSPyzyno4Bkm4+V5pux85tmZma16rYYRMTmiLgjh58A7gPGA7OARdlsEXBSDs8CFkdhFTBK0gHA8cCKiNgaEduAFcCMnLZPRKyKiAAWl5ZlZmYNUNc5A0lNwJuA24FxEbE5Jz0MjMvh8cDG0mxtGesq3lYlXm39cyW1SGppb2+vJ3UzM+tCzcVA0iuBHwKfjojHy9PyE330cm4vExELIqI5IprHjh3b16szMxsyaioGkl5BUQi+HxH/nuFHsouH/Lkl45uAiaXZJ2Ssq/iEKnEzM2uQWq4mEnAZcF9EfK00aRnQcUXQbOC6Uvy0vKpoCvBYdictB6ZLGp0njqcDy3Pa45Km5LpOKy3LzMwaoJbnGbwF+DBwt6S1GfsCMB9YKmkO8BDwgZx2I3AC0Ao8CZwOEBFbJZ0HrM5250bE1hw+A7gcGAnclC8zM2uQbotBRPwc6Oy6/2lV2gdwZifLWggsrBJvAQ7vLhczM+sb/gaymZm5GJiZmYuBmZnhYmBmZrgYmJkZLgZmZoaLgZmZ4WJgZma4GJiZGS4GZmZGbfcmMrMBqmneDS8MPzj/xH7MxAY7HxmYmZmLgZmZuRiYmRk+Z2CDXLnP3Mx6zkcGZmbmYmBmZrU9A3mhpC2S7inFxkhaIWl9/hydcUm6RFKrpLskHVmaZ3a2Xy9pdil+lKS7c55L8jnIZmbWQLUcGVwOzKiIzQNWRsRkYGWOA8wEJudrLnApFMUDOAc4BjgaOKejgGSbj5Xmq1yXmZn1sVqegfwzSU0V4VnA1BxeBNwKfD7ji/M5yKskjZJ0QLZdERFbASStAGZIuhXYJyJWZXwxcBJw085slO3afNLYrPf19GqicRGxOYcfBsbl8HhgY6ldW8a6irdViVclaS7FEQcHHnhgD1M32zX528i2M3b6BHIeBUQv5FLLuhZERHNENI8dO7YRqzQzGxJ6Wgweye4f8ueWjG8CJpbaTchYV/EJVeJmZtZAPe0mWgbMBubnz+tK8bMkLaE4WfxYRGyWtBz4P6WTxtOBsyNiq6THJU0BbgdOA77Rw5xsF+bzBGZ9q9tiIOlKihPA+0tqo7gqaD6wVNIc4CHgA9n8RuAEoBV4EjgdIN/0zwNWZ7tzO04mA2dQXLE0kuLEsU8em5k1WC1XE53ayaRpVdoGcGYny1kILKwSbwEO7y4PG3p8NGDWOP4GspmZ+UZ1NrD4aKB3+DJTq5ePDMzMzEcG1v98NNC3KvevjxSsGhcDsyHGXUhWjYuB9QsfDZgNLC4GZkOYjxKsg4uBNYyPBgY2F4ahzcXA+pQLwODkwjD0uBiYWZdcGIYGFwPrdT4a2HW5MOy6XAysx/ymP7R19vt3kRicXAysLi4A1h0fPQxOLgZWld/0rTf46GHwcDEYgvxGb/3NRWLgcTHYxfiN3gazvvj7dYGpjYtBH/Obs1n/2tn/wXIx2ZXPh6h4OFn/kzQDuBgYBnw3IuZ31b65uTlaWlp6tC6/QZvZYNJbhUfSmohorjZtQBwZSBoGfAt4J9AGrJa0LCLu7d/MzMz6XyOOSAbKw22OBlojYkNEPAssAWb1c05mZkPGgDgyAMYDG0vjbcAxlY0kzQXm5ugfJd3fw/XtD/yhh/P2JedVH+dVH+dVnwGZly7cqbwO6mzCQCkGNYmIBcCCnV2OpJbO+s36k/Oqj/Oqj/Oqz1DLa6B0E20CJpbGJ2TMzMwaYKAUg9XAZEmTJI0ATgGW9XNOZmZDxoDoJoqIHZLOApZTXFq6MCLW9eEqd7qrqY84r/o4r/o4r/oMqbwGzPcMzMys/wyUbiIzM+tHLgZmZrZrFQNJYyStkLQ+f47upN2PJW2XdH1FfJKk2yW1SroqT2Yjafccb83pTX2U1+xss17S7IztLWlt6fUHSV/PaR+R1F6a9tFG5ZXxWyXdX1r/qzLen/trT0k3SPqtpHWS5pfa172/JM3IbWyVNK/K9E63VdLZGb9f0vG1LrMWPc1L0jslrZF0d/48tjRP1d9nA3NrkvRUaf3fKc1zVObcKukSSWpQTh+s+P97XtIROa1R++ttku6QtEPSyRXTOvvfrH9/RcQu8wL+GZiXw/OACztpNw14N3B9RXwpcEoOfwf4mxw+A/hODp8CXNXbeQFjgA35c3QOj67Sbg3wthz+CPDNvtxfXeUF3Ao0V5mn3/YXsCfwjmwzArgNmNmT/UVxMcMDwMG5rN8Ah9WyrcBh2X53YFIuZ1gty+zjvN4EvCaHDwc2leap+vtsYG5NwD2dLPfXwBRAwE0dv9O+zqmizf8EHuiH/dUEvAFYDJxc4/9m3ftrlzoyoLiFxaIcXgScVK1RRKwEnijHsnIeC1xTZf7ycq8BptX5yaSWvI4HVkTE1ojYBqwAZlTk+FrgVRRvcL2hV/LqZrkN3V8R8WRE3AIQxa1N7qD43kpP1HKblM62dRawJCKeiYjfAa25vN649UqP84qIOyPivzO+Dhgpafc6198nuXW2QEkHAPtExKoo3ukW08n/dh/ndGrO25u6zS0iHoyIu4DnK+at+j/Q0/21qxWDcRGxOYcfBsbVMe9+wPaI2JHjbRS3yYDS7TJy+mPZvjfzqnZLjvEVbTo+sZQvAXufpLskXSNpIvXpjby+l4fIXy798wyI/SVpFMUR4MpSuJ79VcvvpLNt7WzeWpbZnZ3Jq+x9wB0R8UwpVu332cjcJkm6U9J/SnprqX1bN8vsy5w6/CVwZUWsEfur3nl7tL8GxPcM6iHpJ8Crq0z6YnkkIkJSw66bbVBepwAfLo3/CLgyIp6R9HGKTzbHlmfo47w+GBGbJO0N/DBzW1zLjH29vyQNp/jHvSQiNmS42/01VEh6PXAhML0U7vHvs5dsBg6MiEclHQX8R+bZ7yQdAzwZEfeUwv29v3rVoCsGEXFcZ9MkPSLpgIjYnIdKW+pY9KPAKEnD85NB+ZYYHbfLaMs3mX2zfW/mtQmYWhqfQNEn2bGMNwLDI2JNaZ3lHL5L0df+En2ZV0Rsyp9PSPoBxSHvYgbA/qL4Ys76iPh6aZ3d7q8q6+juNimdbWtX8+7srVd2Ji8kTQCuBU6LiAc6Zuji99mQ3PKI95nMYY2kB4DXZvtyV1+9+2yn9lc6hYqjggbur67mnVox7630cH/tat1Ey4COM+qzgetqnTH/EG8BOs7Wl+cvL/dk4KcVXTW9kddyYLqk0SqunpmesQ6nUvHHmG+UHd4D3FdHTjuVl6ThkvbPPF4BvAvo+NTUr/tL0vkU/8yfLs/Qg/1Vy21SOtvWZcApKq5SmQRMpjip1xu3XulxXtl1dgPFCfpfdDTu5vfZqNzGqni2CZIOpthnG7LL8HFJU7Ir5jTq+N/emZwyl92AD1A6X9Dg/dWZqv8DPd5f3Z1hHkwvij6+lcB64CfAmIw3Uzw9raPdbUA78BRFf9rxGT+Y4h+2Fbga2D3je+R4a04/uI/y+utcRytwesUyNgCvq4j9E8VJwN9QFLLXNSovYC+KK5vuyhwuBob19/6i+BQUFG/0a/P10Z7uL+AE4L8orvj4YsbOBd7T3bZSdHk9ANxP6WqOasvswd96j/ICvgT8qbRv1lJclNDp77OBub0v172W4sT/u0vLbKZ4s30A+CZ594S+zimnTQVWVSyvkfvrzynep/5EcbSyrrv3jJ7sL9+OwszMdrluIjMz6wEXAzMzczEwMzMXAzMzw8XAzMxwMbBBQtJz+bX/eyRdLWnPAZDTVEn/q8HrbJLUk+vZzbrkYmCDxVMRcUREHA48C3yilpny26R9ZSpQVzHo43zMeszFwAaj24BDJL1bxb3n75T0E0njACR9RdIVkn4BXJGfpm9TcU/4Ozo+zecn+/+UdJ2kDZLmq7h//a9V3Av+z7LdWEk/lLQ6X29Rcb/7TwCfySOWt1ZrVy2f8oZIWiLpxNL45ZJO7iznink/IumbpfHrJU3N4emSfpXzXi3plRmfL+leFTfr+2rv/Ups0OvJN+b88qvRL+CP+XM4xVfr/4biHu4dX5z8KPAvOfwVim+HjszxPYE9cngy0JLDU4HtwAEUzx3YBPxjTvsU8PUc/gHwFzl8IHBfaT1/X8qxq3Yv5FOxXe8FFuXwCIq7UI7sIucm8p7/VDyfAbg+t2l/4GfAXhn/PPAPFN/svr+0z0b19+/Vr4Hz8iGrDRYjJa3N4duAy4BDgavynkMjgN+V2i+LiKdy+BXAN1U8oeo5ipufdVgdebtsFTdGuznjdwPvyOHjgMP04h2K9+n4pF2hq3blfMpuAi5W8UyBGcDPIuIpSft2kXN3plA8XOcXmcsI4FcUt2V+GrhMxVP+ru90CTbkuBjYYPFURBxRDkj6BvC1iFiW3SNfKU3+U2n4M8AjwBspukafLk0r38v/+dL487z4/7EbMCUiyvOhl9++vqt2f6psDBART0u6leJBJX/JizdD6yrnDjt4aVfvHh2rpHjoyamVM0g6muJJfycDZzFEb+FtL+dzBjaY7cuLt+ad3U27zRHxPMU954fVuZ6bgU92jOSndSielrd3De26cxVwOvBW4Md15PwgcISk3VQ8qOfojK8C3iLpkMxjL0mvzaOUfSPiRopi88Ya87MhwMXABrOvAFdLWgP8oYt23wZmS/oN8Do6+ZTehb8FmvOk6728eCXTj4D3dpxA7qJdd24G3g78JIpHH9aa8y8ousbuBS6huNMnEdFOcT7hSkl3UXQRvY6icF2fsZ8Df1frDrBdn+9aamZmPjIwMzMXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM8PFwMzMgP8PscUqwf5NkxgAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.hist(conc2.numpy(), 100, (-0.1,0.1))\n",
+ "plt.title('Parameter Values Histogram') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter values\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"Parameter_Histogram.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 84,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEWCAYAAACNJFuYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAApM0lEQVR4nO3de7wdVXn/8c+XcIsIJEBMIQEDEqURBSFCrNqmBEPAarBFhFoJiEQrXmurwdqCiBX6s3KpF0qFknjhIpYSBYwBjIoaIAhylXIIlyRyCYT7tYHn98d6Npns2eecfU7ONfm+X6/92jNr1sysmT17nllrZq+tiMDMzKxqo8EugJmZDT0ODmZmVuPgYGZmNQ4OZmZW4+BgZmY1Dg5mZlbj4GDWQ5KmSlo+2OVYn0k6UtLVg12OZpIulzRrsMsxEIZNcJD0VOX1kqRnK+Pv76N1bCrpIkn3SApJU5umS9Ipkh7J1ymSlNO2lrRA0mOSvidpRGW+syT9ZTfrvqdpm56StIOkCVmWavrv8iBtjP+fpBcq42e2WP6Rkl7M6U/kMv6iL/ZbX5N0gqTv9uPyz5V0Uj8uPyQ9nft6haSvVY+HoaivA56kcZJWS3pNi2kXS/pqX62rr0haJOlDTWlr7ZeIODAi5raxrJC0a3+Uc6AMm+AQEa9svID7gHdV0r7Xh6u6Gvgb4IEW02YDBwN7AG8E3gV8OKd9GLgBGAtMAN4DIOktwA4R8d9trLu6Ta+MiD9Upo2qpO+RB2ljf3wP+NfK9I90svzfZP5RwDeB8yWNaqNcQ0oG6aF+7O6R+3oa8NfAMT2ZWdLG/VKqftJc3ohYAVwJfKAp3zbAQUC3J1hrbaCOjaH+BeuWpM0knSbpD/k6TdJmOW2qpOWSPi/p4bw677SWEREvRMRpEXE18GKLLLOAf4uI5Xnw/xtwZE7bGfhZRDwP/BLYJa8WTwU+0XdbvO4i4iXgO8AWwER4eT9+VdJ9kh6UdKakkY15JM2UdGPWOu6SNCPTd5A0X9IqSR2SjqnMc4KkCyXNk/SkpFslTa5M/1xeWT8p6Q5J03K5nwfe16glZd5Fkr4s6VfAM5T9e4+k/ZvW993K+Nsk/Tprc8uy9jQbeD/w2Vz+jyrb8UNJKyXdLekTleWMzNrGo5JuA97cg339e8rxsLuk10i6SqXW+bBKDXNUZT335D65CXha0saS5uT+flLSbZLeU8l/pKRfSTo1t3GppD/J9GWSHlKlCaSzz1jSFsDlwA5au9a6UWX9j+RnuU0uq1GjPVrSfcBVLTZ/Lk3BATgMuC0ibu5q26oq69q4krbWVb6kD0q6PT+jBZJenenK/fNQHrs3S9q9zY+vVVleXq+kXSX9XNLj+XlekOm/yOy/y335vkw/Jr8jq/I7s0NludPzO/C4pG/mchvrqX7OjwAntHks/YOkm1RqsWdLGqvS4vCkpCskje5yYyNi2L2Ae4D9c/hEYDHwKmAM8GvgSzltKrAa+BqwGfBnwNPA69pYx3JgalPa48C+lfHJwJM5fCzw/4CRwK+AdwKfBo7v6TY1pU8AAti4i3nPBU7qZvlHAlfn8Igs7wvAqzLtVGA+sA2wJfAj4Cs5bZ/c9ndQLijGAbvltF9QaiGbA3sCK4H9ctoJwHOUK8URwFeAxTntdcAySq2qsZ2vqcz33abyL6LUGF8PbAxs0rzPqvMBrwaeBA7PvNsCe7baX7lN1wP/DGwK7AIsBQ7I6SdTTvDbADsCtwDLu9jXAeyaw5MotdCjgV1zH25GOVZ/AZzWdAzcmOsYmWnvBXbIMr6PcvxuX/lMVwNH5f49KffRN3Id03MfvLKNz3hq8zYBn6R8t8bn8v4DOK/puJxHucgY2WI/jKQcN2+rpP0G+FSb23Z107o2rixnEfChHJ4JdAB/TDk2vgD8OqcdkJ/tKECZZ/tOPreXl1lJW2u/NK33POAfs/ybN23ny8dAju8HPAzslfvy34Ff5LTtgCeAv8zyfxL4v8p6Gp/zx3P6SNo7lhZTWjLGAQ8BvwXelGW9im7OTYN+ou/Ni7WDw13AQZVpBwD3VD7Y1cAWlekXAv/UxjpaBYcXyZNijk/Mg0C5w88CbqKcTMbnh7E1cGZ+eJ2ewHObngIey9f/NH0xHqu8/r5p3nO7WnbTAfZYHnjPAofmNFG+mK+p5H8LcHcO/wdwaotl7pj7ZMtK2leAc3P4BOCKyrRJwLM5vGsesPsDmzQt9wRaB4cTOzsOmucDjgMu7mRfrLW/gH2B+5ryHAf8Vw4vBWZUps2m++DwBPAo5fg8CdioRb6DgRuatueD3XyONwIzK5/pnZVpb8h1j62kPUIJ2t19xlObtwm4HZhWGd8+j52NWXNc7tJNeb8NnFX5vrx8QdLGtrUbHC4Hjq5M24hSu3w15aT8v8CUVp9Bi2PsGdb+rj1F58FhHuU7P76TY6AaHM6mNP02xl+Z+3ICcASlybcxTZQLp2pwuK+bsrc6lt5fGf8h8K3K+MfJc0xnr2HfrES58ri3Mn5vpjU8GhFPN0+XtFOlCv1Um+t6CtiqMr4V8FQUz0XE7Ih4Y0TMoVylfZ7ShLERpdayr7I5phMHR8SofB3cNG27yrTe3sxbHBGjgNGUK8i3Z/oY4BXA9dk88Rjwk0yHEgTuarG8HYBVEfFkJe1eypVKQ/XezTPA5pI2jogO4FOUE/pDks6vVrM7sayb6VWdlbmVV1OOiccq2/95ylUXlO2srvteurdXRIyOiNdExBci4qWs1p+v0pT2BPBdylVj1VrbKOkIlea8Rrl2b5rnwcrwswAR0Zz2Srr/jFt5NXBxJf/tlIuBsZU83X0mc4H3Stqc0sS0ICIeanPb2vVq4PTKclZRTrDjIuIq4OuU2tRDKg+HbNX5ovhE5Xs2CujqoY3P5nquVWky/WAXedc6T0XEU5TAPY6m4yvK2bv54YDm46KdY6n5OGh1XHRqfQgOf6AcHA07ZVrD6GxTXWt6RNwXa9/kbsetlJvRDXtk2loyACgifkK5mluSH/gSyo3sQZUH5t8CH5D0Jkp191ng9ZUvxtaV/bIMqD11QtnP20jaspK2E7CizXJ8PyLeRvn8AjilMamzWZrGn6ac8Br+qDLcWZlbLWcZ5Qp6VOW1ZUQclNPvpwSbhp06WW53/iXX/YaI2Iry4IM6K1u2m/8n8DFg2zxZ3dJinnZ09xm32ufLgAOb9svmUe631crbiaspJ+uZlO2d24tta1zcdfVZf7ipnCMj4tcAEXFGROxNqbm+FviHbsrcloh4ICKOiYgdKA+kfFOdP6G01nkqz0nbUr4r91NaGhrTVB1vrK5pvJ1jaZ2sD8HhPOALksZI2o7Sbtz8GOQXVR5TfTvlSuAHnS1M5abd5jm6qaTN88OCUo38O5XH9HYAPkNpoqjOvzmlWelTmXQ3MFXSpsBbKU0Ugy4iVlGq/P8c5Qb1fwKnSnoVvPwo4gGZ/WzgKJUbxhvltN0iYhnlHs9Xcj+9kdK23u1jqJJeJ2k/lYcHnqOcuF7KyQ8CE9T9E0k3AodJ2kTlRvchlWnfA/aXdKjKjd1tJe1ZWf4ulbzXAk+q3AweKWmEpN0lNW48XwgcJ2m0pPGUKnlvbEmpfT4uaRzdn6S2oJwAVgJIOopydd1jbXzGDwLbStq6MtuZwJe15ubuGEkze7jexn2JUyjt/j/KSW1vW0SspJxE/yY/mw+yduA/k/L5vD6XtbWk9+bwmyXtK2kTSpB5jjXH2TqR9N48HqA0IQZrH8PVY+w8yndozzzm/wW4JiLuAS4F3iDpYJWb7seydvBrpafHUo+tD8HhJMoV+U3AzZR2/uoz7A9QPrg/UE4YH4nyBEln7qCcqMYBC3K4EfH/g3Jw30y5yrk006o+D3wvIpZX5tmO8iVYDlzc4y3sP6cBB+VJ/XOUm3qLs5p6BeWmMRFxLeWm56mUG4w/Z80+OZzSbvoHyrYdHxFXtLHuzShB9GHKZ/QqSjs/rAnej0j6bRfL+CfKSeJR4IvA9xsTIuI+yo3wz1CuXG9kTa3vbGBSNkP8T0S8SLlo2JMSzB+mBM7GifKLlCaBu4GfUp706o0vUm5IPk45drp8vDkibqM8EfcbysnmDZSHHXqrq8/495QT2NLcLzsAp1OaH38q6UnKDc59e7HeeZTa1gVRnubrzbYdQzkBPkJ5KOHXjQkRcTEl+Jyf23ULcGBO3ooSFB+lfIaPUB4c6QtvBq5RaZaeD3wyIhoXfycAc3NfHprfiX+itP3fTzluD8vyP0y5Of+vWb5JlHPa812su0fHUm8ob06sl1R+xPbdiGiuopmZDUlZY15OuaH8s8Eqx/pQczAzG9YkHSBpVDY5fZ5y/2DxYJbJwcHMbPC9hfJ03cOUnhcOjohnB7NA63WzkpmZ9Y5rDmZmVjOsOveq2m677WLChAmDXQwzs2Hj+uuvfzgiuvrh48uGbXCYMGECS5YsGeximJkNG5La+XU/0GazkqRP58/Db5F0Xv7gaWdJ16j0MnhB/sir8SOyCzL9GkkTKss5LtPvqPz4BkkzMq1D0pwebKuZmfWDboND/vruE8DkiNid0vvjYZQfnZwaEbtSfmBydM5yNKU/o10pP5o6JZczKed7PTCD8lPzESrdWn+D8qOVScDhmdfMzAZJuzekNwZG5k+7X0H5hd9+wEU5fS6lV0Aofag0/sjjImBadj8xEzg/Ip6PiLspv9TcJ18dEbE0Il4Azs+8ZmY2SLoNDtnJ1lcp/cTfT/m59vXAYxGxOrMtZ01PnOPIHgRz+uOUDqZeTm+ap7P0GkmzJS2RtGTlypXtbJ+ZmfVCO81KoylX8jtTupbdgtIsNOAi4qyImBwRk8eMaeuGu5mZ9UI7zUr7U7ozXhkR/0fp4OmtwCit+du+8azppnkF2b1xTt+a0pnUy+lN83SWbmZmg6Sd4HAfMEXSK/LewTTgNuBnrOkieRZwSQ7Pz3Fy+lXZbe98SvfKm0namfKvUNcC1wET8+mnTSk3reev+6aZmVlvdfs7h4i4RtJFlK6wVwM3UP4a71JKF7knZdrZOcvZwHckdVC6Sm50S3urpAspgWU1cGx2lYykj1G6xx4BnBMRtT/QMTOzgTNs+1aaPHly+EdwZmbtk3R9RExuJ++w/YW02VAyYc6lLw/fc/I7B7EkZn3DHe+ZmVmNaw5mvVStLZitbxwczPqYm5hsfeDgYNYDri3YhsL3HMzMrMbBwczMahwczMysxvcczPqRb07bcOWag5mZ1bjmYNYNP6FkGyLXHMzMrMbBwczMahwczMysxsHBzMxqHBzMzKzGwcHMzGr8KKvZAPEP4mw46bbmIOl1km6svJ6Q9ClJ20haKOnOfB+d+SXpDEkdkm6StFdlWbMy/52SZlXS95Z0c85zhiT1z+aamVk7ug0OEXFHROwZEXsCewPPABcDc4ArI2IicGWOAxwITMzXbOBbAJK2AY4H9gX2AY5vBJTMc0xlvhl9sXFmZtY7Pb3nMA24KyLuBWYCczN9LnBwDs8E5kWxGBglaXvgAGBhRKyKiEeBhcCMnLZVRCyOiADmVZZlZmaDoKf3HA4DzsvhsRFxfw4/AIzN4XHAsso8yzOtq/TlLdJrJM2m1EbYaaedelh0s/a5ywzb0LVdc5C0KfBu4AfN0/KKP/qwXC1FxFkRMTkiJo8ZM6a/V2dmtsHqSbPSgcBvI+LBHH8wm4TI94cyfQWwY2W+8ZnWVfr4FulmZjZIehIcDmdNkxLAfKDxxNEs4JJK+hH51NIU4PFsfloATJc0Om9ETwcW5LQnJE3Jp5SOqCzLzMwGQVv3HCRtAbwD+HAl+WTgQklHA/cCh2b6ZcBBQAflyaajACJilaQvAddlvhMjYlUOfxQ4FxgJXJ4vMzMbJG0Fh4h4Gti2Ke0RytNLzXkDOLaT5ZwDnNMifQmweztlMTOz/ufuM8zMrMbdZ5gNAnelYUOdaw5mZlbj4GBmZjUODmZmVuN7DmbJXWaYreGag5mZ1Tg4mJlZjYODmZnVODiYmVmNg4OZmdU4OJiZWY2Dg5mZ1fh3DmaDzP0s2VDkmoOZmdU4OJiZWY2Dg5mZ1bQVHCSNknSRpN9Lul3SWyRtI2mhpDvzfXTmlaQzJHVIuknSXpXlzMr8d0qaVUnfW9LNOc8Z+V/SZmY2SNqtOZwO/CQidgP2AG4H5gBXRsRE4MocBzgQmJiv2cC3ACRtAxwP7AvsAxzfCCiZ55jKfDPWbbPMzGxddPu0kqStgT8FjgSIiBeAFyTNBKZmtrnAIuBzwExgXv6X9OKsdWyfeRdGxKpc7kJghqRFwFYRsTjT5wEHA5f3xQaadcU9sZq11k7NYWdgJfBfkm6Q9G1JWwBjI+L+zPMAMDaHxwHLKvMvz7Su0pe3SK+RNFvSEklLVq5c2UbRzcysN9oJDhsDewHfiog3AU+zpgkJgKwlRN8Xb20RcVZETI6IyWPGjOnv1ZmZbbDaCQ7LgeURcU2OX0QJFg9mcxH5/lBOXwHsWJl/fKZ1lT6+RbqZmQ2SboNDRDwALJP0ukyaBtwGzAcaTxzNAi7J4fnAEfnU0hTg8Wx+WgBMlzQ6b0RPBxbktCckTcmnlI6oLMvMzAZBu91nfBz4nqRNgaXAUZTAcqGko4F7gUMz72XAQUAH8EzmJSJWSfoScF3mO7Fxcxr4KHAuMJJyI9o3o83MBlFbwSEibgQmt5g0rUXeAI7tZDnnAOe0SF8C7N5OWczMrP+54z2zIcSd8NlQ4e4zzMysxsHBzMxqHBzMzKzGwcHMzGocHMzMrMbBwczMavwoq21w3BOrWfdcczAzsxoHBzMzq3FwMDOzGgcHMzOrcXAwM7MaBwczM6txcDAzsxoHBzMzq/GP4MyGKP+3gw0m1xzMzKymreAg6R5JN0u6UdKSTNtG0kJJd+b76EyXpDMkdUi6SdJeleXMyvx3SppVSd87l9+R86qvN9TMzNrXk5rDn0fEnhHR+C/pOcCVETERuDLHAQ4EJuZrNvAtKMEEOB7YF9gHOL4RUDLPMZX5ZvR6i8zMbJ2tyz2HmcDUHJ4LLAI+l+nzIiKAxZJGSdo+8y6MiFUAkhYCMyQtAraKiMWZPg84GLh8HcpmthZ3tmfWM+3WHAL4qaTrJc3OtLERcX8OPwCMzeFxwLLKvMszrav05S3SayTNlrRE0pKVK1e2WXQzM+updmsOb4uIFZJeBSyU9PvqxIgISdH3xVtbRJwFnAUwefLkfl+fmdmGqq2aQ0SsyPeHgIsp9wwezOYi8v2hzL4C2LEy+/hM6yp9fIt0MzMbJN0GB0lbSNqyMQxMB24B5gONJ45mAZfk8HzgiHxqaQrweDY/LQCmSxqdN6KnAwty2hOSpuRTSkdUlmVmZoOgnWalscDF+XTpxsD3I+Inkq4DLpR0NHAvcGjmvww4COgAngGOAoiIVZK+BFyX+U5s3JwGPgqcC4yk3Ij2zWgzs0HUbXCIiKXAHi3SHwGmtUgP4NhOlnUOcE6L9CXA7m2U18zMBoB/IW1mZjUODmZmVuPgYGZmNQ4OZmZW4y67zYYBd99tA801BzMzq3FwMDOzGgcHMzOr8T0HW2+5m26z3nPNwczMahwczMysxsHBzMxqHBzMzKzGwcHMzGocHMzMrMbBwczMahwczMyspu3gIGmEpBsk/TjHd5Z0jaQOSRdI2jTTN8vxjpw+obKM4zL9DkkHVNJnZFqHpDl9uH1mZtYLPak5fBK4vTJ+CnBqROwKPAocnelHA49m+qmZD0mTgMOA1wMzgG9mwBkBfAM4EJgEHJ55zcxskLQVHCSNB94JfDvHBewHXJRZ5gIH5/DMHCenT8v8M4HzI+L5iLgb6AD2yVdHRCyNiBeA8zOvmbUwYc6lL7/M+ku7NYfTgM8CL+X4tsBjEbE6x5cD43J4HLAMIKc/nvlfTm+ap7N0MzMbJN0GB0l/ATwUEdcPQHm6K8tsSUskLVm5cuVgF8fMbL3VTs3hrcC7Jd1DafLZDzgdGCWp0avreGBFDq8AdgTI6VsDj1TTm+bpLL0mIs6KiMkRMXnMmDFtFN3MzHqj2y67I+I44DgASVOBv4+I90v6AXAIJWDMAi7JWebn+G9y+lUREZLmA9+X9DVgB2AicC0gYKKknSlB4TDgr/tqA23D4TZ4s76zLv/n8DngfEknATcAZ2f62cB3JHUAqygneyLiVkkXArcBq4FjI+JFAEkfAxYAI4BzIuLWdSiXmZmtox4Fh4hYBCzK4aWUJ42a8zwHvLeT+b8MfLlF+mXAZT0pi5mZ9R//QtrMzGocHMzMrMbBwczMahwczMysxsHBzMxqHBzMzKzGwcHMzGrW5UdwZjbIqr8Kv+fkdw5iSWx945qDmZnVODiYmVmNg4OZmdX4noMNa+6J1ax/uOZgZmY1Dg5mZlbj4GBmZjUODmZmVuPgYGZmNQ4OZmZW021wkLS5pGsl/U7SrZK+mOk7S7pGUoekCyRtmumb5XhHTp9QWdZxmX6HpAMq6TMyrUPSnH7YTjMz64F2ag7PA/tFxB7AnsAMSVOAU4BTI2JX4FHg6Mx/NPBopp+a+ZA0CTgMeD0wA/impBGSRgDfAA4EJgGHZ14zMxsk3QaHKJ7K0U3yFcB+wEWZPhc4OIdn5jg5fZokZfr5EfF8RNwNdAD75KsjIpZGxAvA+ZnXzMwGSVu/kM6r++uBXSlX+XcBj0XE6syyHBiXw+OAZQARsVrS48C2mb64stjqPMua0vftpByzgdkAO+20UztFN9tguIdW60tt3ZCOiBcjYk9gPOVKf7f+LFQX5TgrIiZHxOQxY8YMRhHMzDYIPXpaKSIeA34GvAUYJalR8xgPrMjhFcCOADl9a+CRanrTPJ2lm5nZIGnnaaUxkkbl8EjgHcDtlCBxSGabBVySw/NznJx+VUREph+WTzPtDEwErgWuAybm00+bUm5az++DbTMzs15q557D9sDcvO+wEXBhRPxY0m3A+ZJOAm4Azs78ZwPfkdQBrKKc7ImIWyVdCNwGrAaOjYgXASR9DFgAjADOiYhb+2wLbb3jnljN+l+3wSEibgLe1CJ9KeX+Q3P6c8B7O1nWl4Evt0i/DLisjfKamdkA8C+kzcysxsHBzMxqHBzMzKzGwcHMzGocHMzMrMbBwczMatrqW8nMhhf3s2TryjUHMzOrcXAwM7MaNyvZsOAuM8wGlmsOZmZW4+BgZmY1Dg5mZlbj4GBmZjUODmZmVuPgYGZmNQ4OZmZW085/SO8o6WeSbpN0q6RPZvo2khZKujPfR2e6JJ0hqUPSTZL2qixrVua/U9KsSvrekm7Oec6QpP7YWLMN0YQ5l671MmtHOzWH1cBnImISMAU4VtIkYA5wZURMBK7McYADgYn5mg18C0owAY4H9qX8vejxjYCSeY6pzDdj3TfNzMx6q53/kL4fuD+Hn5R0OzAOmAlMzWxzgUXA5zJ9XkQEsFjSKEnbZ96FEbEKQNJCYIakRcBWEbE40+cBBwOX98kW2rDlq1yzwdOjew6SJgBvAq4BxmbgAHgAGJvD44BlldmWZ1pX6ctbpJuZ2SBpOzhIeiXwQ+BTEfFEdVrWEqKPy9aqDLMlLZG0ZOXKlf29OjOzDVZbwUHSJpTA8L2I+O9MfjCbi8j3hzJ9BbBjZfbxmdZV+vgW6TURcVZETI6IyWPGjGmn6GZm1gvtPK0k4Gzg9oj4WmXSfKDxxNEs4JJK+hH51NIU4PFsfloATJc0Om9ETwcW5LQnJE3JdR1RWZaZmQ2CdrrsfivwAeBmSTdm2ueBk4ELJR0N3AscmtMuAw4COoBngKMAImKVpC8B12W+Exs3p4GPAucCIyk3on0z2sxsELXztNLVQGe/O5jWIn8Ax3ayrHOAc1qkLwF2764sZmY2MPwLaTMzq/E/wZltYKq/H7nn5HcOYklsKHNwsCHFP3wzGxrcrGRmZjUODmZmVuPgYGZmNQ4OZmZW4+BgZmY1Dg5mZlbjR1nNNmD+zYN1xsHBBp1/22A29LhZyczMahwczMysxsHBzMxqHBzMzKzGwcHMzGr8tJINCj+hNPT4sVarcs3BzMxqug0Oks6R9JCkWypp20haKOnOfB+d6ZJ0hqQOSTdJ2qsyz6zMf6ekWZX0vSXdnPOcIamzvyQ1M7MB0k7N4VxgRlPaHODKiJgIXJnjAAcCE/M1G/gWlGACHA/sC+wDHN8IKJnnmMp8zesyM7MB1m1wiIhfAKuakmcCc3N4LnBwJX1eFIuBUZK2Bw4AFkbEqoh4FFgIzMhpW0XE4ogIYF5lWWZmNkh6e89hbETcn8MPAGNzeBywrJJveaZ1lb68RXpLkmZLWiJpycqVK3tZdDMz6846P60UESEp+qIwbazrLOAsgMmTJw/IOq3v+Aml4cNPLllvaw4PZpMQ+f5Qpq8AdqzkG59pXaWPb5FuZmaDqLfBYT7QeOJoFnBJJf2IfGppCvB4Nj8tAKZLGp03oqcDC3LaE5Km5FNKR1SWZWZmg6TbZiVJ5wFTge0kLac8dXQycKGko4F7gUMz+2XAQUAH8AxwFEBErJL0JeC6zHdiRDRucn+U8kTUSODyfJmZ2SDqNjhExOGdTJrWIm8Ax3aynHOAc1qkLwF2764cNjz5PoPZ8OTuM8ysS745vWFy9xlmZlbj4GBmZjVuVrI+5/sM6y83MW04XHMwM7MaBwczM6txs5L1CTclbXjcxLR+c83BzMxqXHOwXnNtwWz95eBgZuvMTUzrHwcH6xHXFsw2DA4OZtanXItYPzg4WLdcW7DecqAYvhwcrCUHBLMNm4ODvcwBwfqTaxHDi4PDBshBwAZbZ8egg8bQ4eCwHnMQsOHGtYuhw8FhGPJJ3zYE7RznDiD9Z8gEB0kzgNOBEcC3I+LkQS7SgPHJ3qx3HED6z5AIDpJGAN8A3gEsB66TND8ibhvckhU+eZsNX335/d2QAs2QCA7APkBHRCwFkHQ+MBPol+Dgk72Z9cZQOHcMVIAaKsFhHLCsMr4c2Lc5k6TZwOwcfUrSHb1c33bAw72ctz+5XD3jcvWMy9UzQ7JcOmWdyvXqdjMOleDQlog4CzhrXZcjaUlETO6DIvUpl6tnXK6ecbl6ZkMv11D5P4cVwI6V8fGZZmZmg2CoBIfrgImSdpa0KXAYMH+Qy2RmtsEaEs1KEbFa0seABZRHWc+JiFv7cZXr3DTVT1yunnG5esbl6pkNulyKiIFYj5mZDSNDpVnJzMyGEAcHMzOrWa+Cg6RtJC2UdGe+j+4k308kPSbpx03pO0u6RlKHpAvy5jiSNsvxjpw+oZ/KNSvz3ClpVqZtKenGyuthSafltCMlraxM+9BAlSvTF0m6o7L+V2X6YO6vV0i6VNLvJd0q6eRK/h7vL0kzchs7JM1pMb3TbZV0XKbfIemAdpfZjt6WS9I7JF0v6eZ8368yT8vPcwDLNkHSs5X1n1mZZ+8sc4ekMyRpgMr0/qbv30uS9sxpA7W//lTSbyWtlnRI07TOvpvrtL8AiIj15gX8KzAnh+cAp3SSbxrwLuDHTekXAofl8JnA3+bwR4Ezc/gw4IK+LhewDbA030fn8OgW+a4H/jSHjwS+3p/7q6tyAYuAyS3mGbT9BbwC+PPMsynwS+DA3uwvysMRdwG75LJ+B0xqZ1uBSZl/M2DnXM6IdpbZz+V6E7BDDu8OrKjM0/LzHMCyTQBu6WS51wJTAAGXNz7T/i5TU543AHcNwv6aALwRmAcc0uZ3s9f7q/Far2oOlC435ubwXODgVpki4krgyWpaRtb9gItazF9d7kXAtB5G4nbKdQCwMCJWRcSjwEJgRlMZXwu8inLC6wt9Uq5uljug+ysinomInwFExAvAbym/m+mNl7t1yWU1unXprKzVbZ0JnB8Rz0fE3UBHLq+dZfZbuSLihoj4Q6bfCoyUtFkP198vZetsgZK2B7aKiMVRznzz6OS73c9lOjzn7Uvdli0i7omIm4CXmuZt+R3og/0FrGfNSsDYiLg/hx8AxvZg3m2BxyJidY4vp3TrAZXuPXL645m/L8vVqguRcU15Glc01UfM/krSTZIukrQjPdMX5fqvrFL/U+XLNCT2l6RRlBrilZXknuyvdj6Tzra1s3nbWWZ31qVcVX8F/DYinq+ktfo8B7JsO0u6QdLPJb29kn95N8vszzI1vA84ryltIPZXT+dd1/0FDJHfOfSEpCuAP2ox6R+rIxERkgbsOd0BKtdhwAcq4z8CzouI5yV9mHLls191hn4u1/sjYoWkLYEfZtnmtTNjf+8vSRtTvshnRHboSBv7a0Mh6fXAKcD0SnKvP88+cj+wU0Q8Imlv4H+ynINO0r7AMxFxSyV5sPdXvxp2wSEi9u9smqQHJW0fEfdn1eqhHiz6EWCUpI3zyqHahUeje4/ledLZOvP3ZblWAFMr4+MpbZqNZewBbBwR11fWWS3Dtylt9Wvpz3JFxIp8f1LS9ylV5HkMgf1F+aHQnRFxWmWd3e6vFuvorluXzra1q3nXtauYdSkXksYDFwNHRMRdjRm6+DwHpGxZI34+y3C9pLuA12b+atNgT/fZOu2vdBhNtYYB3F9dzTu1ad5FrPv+Ata/ZqX5QOOO/SzgknZnzAPzZ0DjaYDq/NXlHgJc1dS00xflWgBMlzRa5emc6ZnWcDhNB2eeOBveDdzegzKtU7kkbSxpuyzHJsBfAI2rqkHdX5JOony5P1WdoRf7q51uXTrb1vnAYSpPwewMTKTcJOyLrmJ6Xa5saruUcsP/V43M3XyeA1W2MSr/7YKkXSj7bGk2MT4haUo23RxBD77b61KmLMtGwKFU7jcM8P7qTMvvQB/sr6Knd7CH8ovSRnglcCdwBbBNpk+m/LtcI98vgZXAs5T2uAMyfRfKF7gD+AGwWaZvnuMdOX2XfirXB3MdHcBRTctYCuzWlPYVyk3F31EC224DVS5gC8qTUzdlGU4HRgz2/qJcJQXlxH9jvj7U2/0FHAT8L+WJkn/MtBOBd3e3rZQmsruAO6g8LdJqmb041ntVLuALwNOVfXMj5SGHTj/PASzbX+W6b6Q8SPCuyjInU06+dwFfJ3t36O8y5bSpwOKm5Q3k/noz5Tz1NKU2c2t354x13V8R4e4zzMysbn1rVjIzsz7g4GBmZjUODmZmVuPgYGZmNQ4OZmZW4+Bgw4KkF7Obglsk/UDSK4ZAmaZK+pMBXucESb15nt6sRxwcbLh4NiL2jIjdgReAj7QzU/7atb9MBXoUHPq5PGZ9xsHBhqNfArtKepdK3/s3SLpC0lgASSdI+o6kXwHfyavtX6r0if/bxtV+Xvn/XNIlkpZKOlml//5rVfrCf03mGyPph5Kuy9dbVfr7/wjw6azRvL1VvlblqW6IpPMlvbMyfq6kQzorc9O8R0r6emX8x5Km5vB0Sb/JeX8g6ZWZfrKk21Q6H/xq330ktr7xVYwNK3nlfSDwE+BqYEpEhMof93wW+ExmnQS8LSKezSaod0TEc5ImUrohmZz59gD+GFhF+RX6tyNiH0mfBD5O6YLjdODUiLha0k6ULgr+WOXPaJ6KiK9m2b7fnC+XvVZ5mjbpAkrXDJdm9wnTgL+l9MPfWZm720fbUX4JvX9EPC3pc8DfSfoG8B7KL8MbXWmYteTgYMPFSEk35vAvgbOB1wEXqPSZtClwdyX//MqJeBPg6yr/4PUipTO3husiuwdX6ejtp5l+M/DnObw/MElremTeqnEl3qSrfPNbBAYof8Ryusp/KswAfpEBbesuytydKZRg9Kssy6bAbyjdUD8HnK3yL4g/7nQJtsFzcLDh4tmI2LOaIOnfga9FxPxsTjmhMvnpyvCngQcptYSNKCfIhup/GbxUGX+JNd+PjSg1lOp8qN59f1f5nm7ODJA1g0WUP255H2s6d+uqzA2rWbtpePPGKil/AnN48wyS9qHUTg4BPsYG2mW5dc/3HGw425o1XRHP6ibf/RHxEqXP/RE9XM9PKU1MAOTVPJR/E9yyjXzduQA4Cng7pbms3TLfA+wpaSOVPy7aJ9MXA2+VtGuWYwtJr81azNYRcRkl+OzRZvlsA+TgYMPZCcAPJF0PPNxFvm8CsyT9DtiNTq7iu/AJYHLexL2NNU9K/Qh4T+OGdBf5uvNT4M+AK6L8VWS7Zf4VpSntNuAMSk+mRMRKyv9lnyfpJkqT0m6UQPbjTLsa+Lt2d4BteNwrq5mZ1bjmYGZmNQ4OZmZW4+BgZmY1Dg5mZlbj4GBmZjUODmZmVuPgYGZmNf8fMmWWIZN/rYEAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.hist(reverse_top10.numpy(), 100, (-0.1,0.1))\n",
+ "plt.title('Top-10% FFT Reconstructed Parameter Values Histogram') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter values\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"FFT_Histogram.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 85,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEWCAYAAACKSkfIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAfa0lEQVR4nO3dfbxcVX3v8c+XhPAkkGBigIRwoES4aAX1lOClagTKoxisqKEqgeKNtthqpS8JqFURXoVeLWCpYipIoAoErBJ5EAJCBRQl4UkeSjlEKIk8BJIgSIAb+N0/1hrYGc+cM3POnDnnZH3fr9e8Zu+119577TUzv1mz9p61FRGYmVkZNhruApiZWec46JuZFcRB38ysIA76ZmYFcdA3MyuIg76ZWUEc9M1GIEkzJS0f7nLUk3SOpC8Odzls4Bz0h4Gk5yqPVyStrcx/pE37GCfpMkkPSwpJM+uWS9Lpkp7Oj9MlKS/bWtI1ktZI+p6kMZX15kv68372/XDlmJ6QdL6k17XjuIaKpK5cT2PbtL1Nc/3t28uyMyRd1o79tFN+nU6pS1uvXiLikxHx1Sa29bCk/YeqrDZwDvrDICJeV3sA/wMcVkn7Xht3dTPwUeDxXpbNBQ4H9gDeAhwGfCIv+wRwBzAZ6ALeDyDpHcD2EfEfTez7sHx8bwO6gS+0UvD8pTRq3p/1XxYR8QJwCXBUXb4xwJHAgs6VbsPSri/mUo2aD1UJJG0i6UxJv82PMyVtkpfNlLRc0kmSnsotqYa/CiLipYg4MyJuBl7uJcsc4OsRsTwiVgBfB47Oy3YCboiIF4GbgJ1zsDoD+NtWjilv+2rgzZImSLpC0kpJq/P01Mrx3yjpVEm3AM/n/R4j6X5Jz0paJukTlfy1OvmcpCclPSbpcEmHSPpvSasknVTJv5GkeZIeyr9uFkraJi/+WX5ek3+hvCOv85d5/6vzr58dK9sLScdJehB4sJfDXwB8QNLmlbQDSZ+7q/s6tnp5X7tU5tdrlUt6r6Q786+Ln0t6S2XZCZJW5P08IGm/RvvpT3W/kibm13BNruubch1fCEwDfpzr8nM5//sk3Zvz3yjpf1W2+zZJd+QyXirpksp+aq/zCZIeB77b5HvplFwXz0n6saTXK/1y/Z2k2yR1DbQeRjMH/ZHl88DewJ6kFvherN9C3haYCEwhBe35knYd4L7eBNxVmb8rpwHcA+wvaTPgncC9pGB/dUQsa2UnknYADiH9ctgI+C6wIykorAXOrlvlY6RfIVsCjwBPAu8FtgKOAc6Q9LZK/m2BTUl18g/Av5F+3bw9l/2LknbKef+G9Ovm3cD2wGrgX/Oyd+Xn8fkX1y8kzQJOAv4cmET6AryorryHAzOA3euPPSJ+DjyW168e3/cjYl0Tx9YUSW8FziP9Qns98G1gUW5E7Ap8CviTiNiS9KXzcKv7aOB4YDmpbiaT6ioi4mOs/wv2nyS9kVR3n8n5ryJ9KYyTNA74IXA+sE3O9/66fW2bl+1Ien80816aTarvKcAfAb/I62wD3A98qR2VMOpEhB/D+CB9APfP0w8Bh1SWHQg8nKdnAuuALSrLFwJfbGIfy4GZdWkvA7tV5qcDAYgUROcDdwOnAVOB24GtgXNIreJT+jmm54A1pMD9TWCzXvLtCayuzN8InNzPsfwI+HSlTtYCY/L8lvkYZlTyLwUOz9P3A/tVlm0H/D9gLKkbK4CxleVXA8dW5jci/QLZMc8HsG8/5f0CcG2e3iqv/9Ymj215ZVkAu1Tmz6+9BsC3gK/WbesB0pfbLqQvl/2Bjfsp6/nAC/l1qz1+V62Xuv2eDFxeLVdv7+s8/0VgYV1drsjH+a48rcrymyv7mQm8BGzaR9l7ey99vjL/dVKjpTZ/GHBnOz/Lo+Xhlv7Isj0pSNY8ktNqVkfE7+uXS5qmysnhJvf1HCkI1WwFPBfJCxExNyLeEhHzSN06JwEfIX1Y3w3MkHRQH9s/PCLGR8SOEfHXEbFW0uaSvi3pEUm/I315jFflRDHwaHUjkg6WdGvuPlhD+tUwsZLl6YiodV+tzc9PVJavBWonkXcEfpi7F9aQvgReJrVSe7MjcFYl/yrSl+KURuXtxYXAeyRtDxwBPBQRdzR5bM3aETi+Vs68rR1I5196SK3rLwNPSro4l6WRr+XXbXxEjCed72nk/wI9wLW5e2peH3nXe29HxCukupuSl62IHI2z+npdGek8CQBNvpfq3weN3hdFcdAfWX5L+gDXTMtpNRMkbVG/PCL+J9Y/OdyMe0ldSDV75LT15MCuiPgJ8MfAkvzhXELfAaE3xwO7klriW/Fal4oqeV794Cudz/gB8DVgcg5CV9Xlb8WjwMHVoBYRm0Y679DbcLOPAp+oy79ZpG6bPyhvbyLiEVK30EdJXQ0LBnhszwPVcwPb1pXz1Lpybh4RF+UyfD8i/pT03grg9L7K3KyIeDYijo+InYH3AZ+tnC+or5f13tuSRPpiWkHqApuS02p2qN9d3Xwz7yXrhYP+yHIR8AVJkyRNJPVR/3tdnq/kftB3kvqDL220sdynu2meHad0GWHtQ3EB6UM6Jbf8jif9dK+uvympe+czOek3wMzcB7sP0FL/Pqn7ZS3pZOk29N+nOg7YBFgJrJN0MHBAi/usOgc4VflkbK7nWXnZSuAVYOe6/CdKelPOv7WkDw5gvwtI/er7ALWrs1o9tjuBv5A0Jn8Rv7uy7N+AT0qaoWQLSYdK2lLSrpL2zV8yL5Dq/5UBHMMfUDp5vEt+Tz1D+tVU2/YTrF+XC4FDJe0naWPS++1F4OekvvaXgU9JGptfk7362X2r7yXLHPRHllNILei7gV+T+tGr100/Tjr5+FtS8PhkRPxXH9t7gPTBmAJck6drra1vAz/O+7kHuDKnVZ0EfC8illfWmUgKVMtJJ99acSawGfAUcCvwk74yR8SzpBPIC0nH/RfAohb3WXVWXv9aSc/mMszI+3oeOBW4JXeR7B0RPyS1ii/OXQj3AAcPYL8/IJ08vD4iHhvgsX2a1A+9htTN9qPagohYAvwf0onM1aQul6Pz4k1IX9xPkd4/bwBOHMAx9GY6cB2pq/AXwDcj4oa87B9JDZg1kv4+Ih4g/dr5l1yWw0gnel+KiJdIJ7uPzcf3UeAK0pdCI2fSwnvJXqP1u9FspFL6c9W/R8TUfrKajXqSfgmcExHfHe6ybGjc0jezYSfp3ZK2zd07c0jni9x6HwL+Z5uZjQS7krq6tiCdKzqi1hVm7eXuHTOzgrh7x8ysICO6e2fixInR1dU13MUwMxtVli5d+lRETOpt2YgO+l1dXSxZsmS4i2FmNqpIeqTRMnfvmJkVxEHfzKwgDvpmZgVx0DczK4iDvplZQZoK+kq35vu10u3YluS0bSQtlvRgfp6Q0yXpG5J6JN2typ2AJM3J+R/Mf7U2M7MOaqWl/56I2DMiuvP8PNKogdOB6/M8pFEIp+fHXNJdfagMfzqDNGzql2pfFGZm1hmD6d6ZRb4hRH4+vJJ+Qb4D062ku9lsR7r13+KIWBURq4HFQF93XjIzszZrNugHaQzypZLm5rTJlQGRHue1W85NYf1bnS3PaY3SzcysQ5r9R+6fRsQKSW8AFkta78YdERGS2jJyW/5SmQswbdq0dmzSOqxr3pWvTj982qHDWBIzq9dUSz/fQ5SIeJJ0t6S9gCdytw35+cmcfQXr399yak5rlF6/r/kR0R0R3ZMm9Tp0hJmZDVC/QT/fb3PL2jTpPp73kG7tVrsCZw5weZ5eBByVr+LZG3gmdwNdAxwgaUI+gXtATjMzsw5ppntnMvDDfD/tscD3I+Inkm4DFko6FngE+FDOfxVwCOk+nc8DxwBExCpJXwVuy/lOjohVbTsSMzPrV79BPyKWAXv0kv40sF8v6QEc12Bb5wHntV5MMzNrB/8j18ysIA76ZmYFcdA3MyuIg76ZWUEc9M3MCuKgb2ZWEAd9M7OCOOibmRXEQd/MrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgzdwu0axfXfOuHO4imFkT3NI3MyuIg76ZWUEc9M3MCuKgb2ZWEAd9M7OCOOibmRXEQd/MrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgTQd9SWMk3SHpijy/k6RfSuqRdImkcTl9kzzfk5d3VbZxYk5/QNKBbT8aMzPrUyst/U8D91fmTwfOiIhdgNXAsTn9WGB1Tj8j50PS7sBs4E3AQcA3JY0ZXPHNzKwVTQV9SVOBQ4Hv5HkB+wKX5SwLgMPz9Kw8T16+X84/C7g4Il6MiN8APcBebTgGMzNrUrMt/TOBzwGv5PnXA2siYl2eXw5MydNTgEcB8vJncv5X03tZ51WS5kpaImnJypUrmz8SMzPrV7/3yJX0XuDJiFgqaeZQFygi5gPzAbq7u2Oo92cD5/vimo0+zdwYfR/gfZIOATYFtgLOAsZLGptb81OBFTn/CmAHYLmkscDWwNOV9JrqOmZm1gH9du9ExIkRMTUiukgnYn8aER8BbgCOyNnmAJfn6UV5nrz8pxEROX12vrpnJ2A68Ku2HYmZtU3XvCtffdiGpZmWfiMnABdLOgW4Azg3p58LXCipB1hF+qIgIu6VtBC4D1gHHBcRLw9i/2Zm1qKWgn5E3AjcmKeX0cvVNxHxAvDBBuufCpzaaiHNzKw9/I9cM7OCOOibmRXEQd/MrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgDvpmZgUZzCibZlaA6vDKD5926DCWxNrBLX0zs4K4pW8t8U01zEY3t/TNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgDvpmZgVx0DczK4iv07d++dp8sw2HW/pmZgVxS9/MmuZxeEY/t/TNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgDvpmZgVx0DczK0i/QV/SppJ+JekuSfdK+kpO30nSLyX1SLpE0ricvkme78nLuyrbOjGnPyDpwCE7KjMz61Uzf856Edg3Ip6TtDFws6Srgc8CZ0TExZLOAY4FvpWfV0fELpJmA6cDH5a0OzAbeBOwPXCdpDdGxMtDcFw2SB56wWzD1G9LP5Ln8uzG+RHAvsBlOX0BcHienpXnycv3k6ScfnFEvBgRvwF6gL3acRBmZtacpvr0JY2RdCfwJLAYeAhYExHrcpblwJQ8PQV4FCAvfwZ4fTW9l3Wq+5oraYmkJStXrmz5gMzMrLGmgn5EvBwRewJTSa3z3YaqQBExPyK6I6J70qRJQ7UbM7MitXT1TkSsAW4A3gGMl1Q7JzAVWJGnVwA7AOTlWwNPV9N7WcfMzDqgmat3Jkkan6c3A/4MuJ8U/I/I2eYAl+fpRXmevPynERE5fXa+umcnYDrwqzYdh5mZNaGZq3e2AxZIGkP6klgYEVdIug+4WNIpwB3AuTn/ucCFknqAVaQrdoiIeyUtBO4D1gHH+codM7PO6jfoR8TdwFt7SV9GL1ffRMQLwAcbbOtU4NTWi2lmZu3gm6iY2YD4hiqjk4dhMDMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxJds2qs8nLLZhs8tfTOzgjjom5kVxEHfzKwgDvpmZgXxiVwzGzSPwzN6uKVvZlYQB30zs4K4e8fM2spdPSObg37h/Icss7K4e8fMrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgvg6fTMbMv6j1sjjoF8g/yHLrFzu3jEzK4iDvplZQRz0zcwK4qBvZlYQn8g1s47wlTwjg1v6ZmYFcUu/EL5M08ygiaAvaQfgAmAyEMD8iDhL0jbAJUAX8DDwoYhYLUnAWcAhwPPA0RFxe97WHOALedOnRMSC9h6O1TjIm1lvmmnprwOOj4jbJW0JLJW0GDgauD4iTpM0D5gHnAAcDEzPjxnAt4AZ+UviS0A36ctjqaRFEbG63QdlZqOH+/o7q98+/Yh4rNZSj4hngfuBKcAsoNZSXwAcnqdnARdEciswXtJ2wIHA4ohYlQP9YuCgdh6MmZn1raU+fUldwFuBXwKTI+KxvOhxUvcPpC+ERyurLc9pjdLr9zEXmAswbdq0VopnZqOEux+HT9NBX9LrgB8An4mI36Wu+yQiQlK0o0ARMR+YD9Dd3d2WbZbCHyQz609Tl2xK2pgU8L8XEf+Rk5/I3Tbk5ydz+gpgh8rqU3Nao3QzM+uQfoN+vhrnXOD+iPjnyqJFwJw8PQe4vJJ+lJK9gWdyN9A1wAGSJkiaAByQ08zMrEOa6d7ZB/gY8GtJd+a0k4DTgIWSjgUeAT6Ul11Fulyzh3TJ5jEAEbFK0leB23K+kyNiVTsOwsw2DL6SZ+j1G/Qj4mZADRbv10v+AI5rsK3zgPNaKaCZmbWP/5FrZiOSW/1Dw2PvmJkVxEHfzKwg7t4ZhXw9vpkNlFv6ZmYFcUvfzEYVn+AdHLf0zcwK4qBvZlYQd++Y2YjX6OIFd/W0zi19M7OCuKU/SvgyTTNrB7f0zcwK4pb+CObWvZm1m4P+CONAb2ZDyUHfzDZovsJnfQ76ZrZBcHBvjk/kmpkVxC39YVJKq6SU47SRxefGGnNL38ysIG7pjwBulZh1hn95uqVvZlYUt/SHmFsWZiNTqZ9NB/0OcjeOmQ03B/0h4OBuNnpt6L8AHPTbxIHezEYDB30zK15JjTZfvWNmVhC39AehpNaBmW0YHPTNzA2YBvqql9F6ktdBv0X+cJjZaOagb2Y2AI0agCP9F4CDfkWj63PdujezDUW/V+9IOk/Sk5LuqaRtI2mxpAfz84ScLknfkNQj6W5Jb6usMyfnf1DSnKE5HDOz4dU178pXHyNRMy3984GzgQsqafOA6yPiNEnz8vwJwMHA9PyYAXwLmCFpG+BLQDcQwFJJiyJidbsOpN1G6gtmZqPHSPx3b79BPyJ+JqmrLnkWMDNPLwBuJAX9WcAFERHArZLGS9ou510cEasAJC0GDgIuGvwhNGe09r+ZmbXTQP+cNTkiHsvTjwOT8/QU4NFKvuU5rVH6H5A0V9ISSUtWrlw5wOKZmVlvBn0iNyJCUrSjMHl784H5AN3d3W3bbiPuxjGzkgw06D8habuIeCx33zyZ01cAO1TyTc1pK3itO6iWfuMA9900B3QzGylGSv/+QLt3FgG1K3DmAJdX0o/KV/HsDTyTu4GuAQ6QNCFf6XNATjMzK1qnr/bpt6Uv6SJSK32ipOWkq3BOAxZKOhZ4BPhQzn4VcAjQAzwPHAMQEaskfRW4Lec7uXZS18ysNMPZC9HM1TtHNli0Xy95AziuwXbOA85rqXRmZgXpRBeQh1Y2MyuIg76ZWUEc9M3MCuKgb2ZWEAd9M7OCOOibmRXEQd/MrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgDvpmZgVx0DczK4iDvplZQRz0zcwK4qBvZlYQB30zs4I46JuZFcRB38ysIA76ZmYFcdA3MyuIg76ZWUEc9M3MCuKgb2ZWEAd9M7OCOOibmRXEQd/MrCAO+mZmBXHQNzMrSMeDvqSDJD0gqUfSvE7v38ysZB0N+pLGAP8KHAzsDhwpafdOlsHMrGSdbunvBfRExLKIeAm4GJjV4TKYmRVrbIf3NwV4tDK/HJhRzSBpLjA3zz4n6YFB7G8i8NQg1h8qRZZLpw941SLraxBcrtaMyHLp9EGVa8dGCzod9PsVEfOB+e3YlqQlEdHdjm21k8vVGperNS5Xa0orV6e7d1YAO1Tmp+Y0MzPrgE4H/duA6ZJ2kjQOmA0s6nAZzMyK1dHunYhYJ+lTwDXAGOC8iLh3CHfZlm6iIeBytcblao3L1ZqiyqWIGIrtmpnZCOR/5JqZFcRB38ysIKMu6EvaRtJiSQ/m5wkN8v1E0hpJV9Sl7yTpl3kYiEvyCWUkbZLne/LyriEq15yc50FJc3LalpLurDyeknRmXna0pJWVZR/vVLly+o152Iza/t+Q04ezvjaXdKWk/5J0r6TTKvkHVF/9DQ/S1/FKOjGnPyDpwGa3OZTlkvRnkpZK+nV+3reyTq+vaYfK1SVpbWXf51TWeXsub4+kb0hSB8v1kbrP4CuS9szLOlFf75J0u6R1ko6oW9boszmw+oqIUfUA/gmYl6fnAac3yLcfcBhwRV36QmB2nj4H+Ks8/dfAOXl6NnBJu8sFbAMsy88T8vSEXvItBd6Vp48Gzh7K+uqrXMCNQHcv6wxbfQGbA+/JecYBNwEHD7S+SBcVPATsnLd3F7B7M8dLGk7kLmATYKe8nTHNbHOIy/VWYPs8/WZgRWWdXl/TDpWrC7inwXZ/BewNCLi69pp2olx1ef4YeKjD9dUFvAW4ADiiyc/mgOpr1LX0ScM2LMjTC4DDe8sUEdcDz1bT8jfhvsBlvaxf3e5lwH4ttjSaKdeBwOKIWBURq4HFwEF1ZXwj8AZSIGuHtpSrn+12tL4i4vmIuAEg0nAet5P+8zFQzQwP0uh4ZwEXR8SLEfEboCdvrx1Djgy4XBFxR0T8NqffC2wmaZMW99/2cjXaoKTtgK0i4tZIEe0CGny2O1CuI/O67dJvuSLi4Yi4G3ilbt1ePwODqa/RGPQnR8RjefpxYHIL674eWBMR6/L8ctLQEFAZIiIvfybnb2e5ehuGYkpdnlrro3pZ1Qck3S3pMkk70Jp2lOu7+WftFysfkBFRX5LGk37RXV9JbrW+mnldGh1vo3Wb2eZQlqvqA8DtEfFiJa2317RT5dpJ0h2S/lPSOyv5l/ezzaEuV82HgYvq0oa6vlpdd8D1NeKGYQCQdB2wbS+LPl+diYiQ1LFrTjtUrtnAxyrzPwYuiogXJX2C1ErZt7rCEJfrIxGxQtKWwA9y2S5oZsWhri9JY0kfzm9ExLKc3G99lUTSm4DTgQMqyQN+TdvgMWBaRDwt6e3Aj3IZRwRJM4DnI+KeSvJw1lfbjcigHxH7N1om6QlJ20XEY/knzpMtbPppYLyksflbvjoMRG2IiOU5mGyd87ezXCuAmZX5qaT+wto29gDGRsTSyj6rZfgOqS98PUNZrohYkZ+flfR90k/VCxgB9UX688qDEXFmZZ/91leD/fQ3PEij4+1r3cEOOTKYciFpKvBD4KiIeKi2Qh+v6ZCXK/+CfTHvf6mkh4A35vzVLrqO11c2m7pWfofqq691Z9ateyODqK/R2L2zCKidwZ4DXN7sivkNdwNQOzteXb+63SOAn9Z1sbSjXNcAB0iaoHS1ygE5reZI6t5wOSDWvA+4v4UyDapcksZKmpjLsTHwXqDWAhrW+pJ0CukD+5nqCgOsr2aGB2l0vIuA2UpXhewETCedYGvHkCMDLlfu9rqSdLL8llrmfl7TTpRrktJ9NZC0M6m+luWuvt9J2jt3nxxFC5/twZYrl2cj4ENU+vM7WF+N9PoZGFR9NXO2dyQ9SP1v1wMPAtcB2+T0buA7lXw3ASuBtaT+rgNz+s6kD2UPcCmwSU7fNM/35OU7D1G5/jLvowc4pm4by4Dd6tL+kXQi7i7SF9ZunSoXsAXpSqK7cxnOAsYMd32RWjVBCuh35sfHB1NfwCHAf5Ousvh8TjsZeF9/x0vqrnoIeIDKFRS9bXMA7/cBlQv4AvD7Sv3cSbpAoOFr2qFyfSDv907SCfjDKtvsJgXUh4CzySMGdKJcedlM4Na67XWqvv6EFKd+T/rlcW9/MWOg9eVhGMzMCjIau3fMzGyAHPTNzArioG9mVhAHfTOzgjjom5kVxEHfRhRJL+e/u98j6VJJm4+AMs2U9L87vM8uSa1eD27WLwd9G2nWRsSeEfFm4CXgk82slP9dOVRmAi0F/SEuj9mAOejbSHYTsIukw5TGPr9D0nWSJgNI+rKkCyXdAlyYW8c3KY1LfnutdZ5b6v8p6XJJyySdpjR++q+UxiP/o5xvkqQfSLotP/ZRGm/9k8Df5V8g7+wtX2/lqR6IpIslHVqZP1/SEY3KXLfu0ZLOrsxfIWlmnj5A0i/yupdKel1OP03SfUoDz32tfS+JjXoD+ZegH34M1QN4Lj+PJf2t/K9I44jX/kj4ceDrefrLpH9LbpbnNwc2zdPTgSV5eiawBtiONO79CuAredmngTPz9PeBP83T04D7K/v5+0oZ+8r3annqjuv9wII8PY40cuJmfZS5izzuPHX3CACuyMc0EfgZsEVOPwH4B9K/nR+o1Nn44X5d/Rg5D/8EtZFmM0l35umbgHOBXYFL8rg644DfVPIvioi1eXpj4GylOx69TBrIq+a2yEM5Kw3ydW1O/zXwnjy9P7C7Xhs5d6tay7lOX/mq5am6GjhLaUz7g4CfRcRaSVv3Ueb+7E26icstuSzjgF+Qhgt+AThX6c5xVzTcghXHQd9GmrURsWc1QdK/AP8cEYtyt8aXK4t/X5n+O+AJYA9S1+ULlWXVseRfqcy/wmufg42AvSOiuh76w+HT+8r3+/rMABHxgqQbSTfF+DCvDerVV5lr1rF+V+ymtV2SbrBxZP0KkvYi3T3uCOBTFDy8tK3Pffo2GmzNa8PGzukn32MR8QppzPMxLe7nWuBvajO59Q3pDmxbNpGvP5cAxwDvBH7SQpkfBvaUtJHSTWH2yum3AvtI2iWXYwtJb8y/OraOiKtIXyp7NFk+K4CDvo0GXwYulbQUeKqPfN8E5ki6C9iNBq3uPvwt0J1Pft7Ha1cO/Rh4f+1Ebh/5+nMt8G7guki3zWu2zLeQurTuA75BGp2SiFhJ6u+/SNLdpK6d3UhfUFfktJuBzzZbAbbh8yibZmYFcUvfzKwgDvpmZgVx0DczK4iDvplZQRz0zcwK4qBvZlYQB30zs4L8f+sJ3rihP1HBAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.hist(top10_og[top10_og.abs() >0].numpy(), 100, (-0.1,0.1))\n",
+ "plt.title('Top-10% Parameter Values Histogram') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter values\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"top10_Histogram.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 86,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([-0.14400896, -0.04817872, 0.20703338, ..., 0.0729612 ,\n",
+ " -0.06001848, -0.03798665], dtype=float32)"
+ ]
+ },
+ "execution_count": 86,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "top10_og[top10_og.abs() >0].numpy()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Per Layer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 87,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat5000 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat5000.append(v.flatten())\n",
+ "conc500 = torch.cat(flat5000)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 88,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 88,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc500, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 134,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(3.8334)\n",
+ "tensor(0.5911)\n",
+ "tensor(14.2745)\n",
+ "tensor(0.2714)\n",
+ "tensor(29.4115)\n",
+ "tensor(0.4823)\n",
+ "tensor(9.3226)\n",
+ "tensor(0.3447)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "errs = []\n",
+ "lens = []\n",
+ "fft_layers = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " lens.append(len(flat))\n",
+ " flat_fft = fft.rfft(flat)\n",
+ " topk = torch.topk(\n",
+ " flat_fft.abs(), round(0.1*len(flat_fft)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ " top10[topk.indices] = flat_fft[topk.indices]\n",
+ " reverse_top10 = fft.irfft(top10)\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " print(err)\n",
+ " errs.append(err*err)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 135,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[800, 32, 51200, 64, 1605632, 512, 31744, 62]"
+ ]
+ },
+ "execution_count": 135,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lens"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 136,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1171.1743"
+ ]
+ },
+ "execution_count": 136,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 137,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "34.222424"
+ ]
+ },
+ "execution_count": 137,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 93,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fft_conc = torch.cat(fft_layers)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 94,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'conc5000' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
+ "Input \u001b[0;32mIn [94]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m torch\u001b[38;5;241m.\u001b[39mnorm(fft_conc \u001b[38;5;241m-\u001b[39m \u001b[43mconc5000\u001b[49m\n\u001b[1;32m 2\u001b[0m ,\u001b[38;5;241m2\u001b[39m)\n",
+ "\u001b[0;31mNameError\u001b[0m: name 'conc5000' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "torch.norm(fft_conc - conc5000\n",
+ " ,2)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Almost no difference in layerwise vs over the entire weight"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 138,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "dict_keys(['0', '500', '1000', '1500', '2000', '2500', '3000', '3500', '4000', '4500', '5000', '5500', '6000', '6500', '7000', '7500', '8000', '8500', '9000', '9500', '10000', '10500', '11000', '11500', '12000', '12500', '13000', '13500', '14000', '14500', '15000', '15500', '16000', '16500', '17000'])"
+ ]
+ },
+ "execution_count": 138,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "weights.keys()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 139,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(3.8017)\n",
+ "tensor(0.6525)\n",
+ "tensor(9.6627)\n",
+ "tensor(0.2314)\n",
+ "tensor(14.0121)\n",
+ "tensor(0.3602)\n",
+ "tensor(6.0212)\n",
+ "tensor(0.3187)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "errs = []\n",
+ "lens = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " lens.append(len(flat))\n",
+ " topk = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat))\n",
+ " top10[topk.indices] = flat[topk.indices]\n",
+ " err = torch.norm(top10 - flat, 2)\n",
+ " print(err)\n",
+ " errs.append(err*err)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 140,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[800, 32, 51200, 64, 1605632, 512, 31744, 62]"
+ ]
+ },
+ "execution_count": 140,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lens"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 141,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "341.1233"
+ ]
+ },
+ "execution_count": 141,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 142,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "18.469522"
+ ]
+ },
+ "execution_count": 142,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 99,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat5000 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat5000.append(v.flatten())\n",
+ "conc5000 = torch.cat(flat5000)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 100,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 100,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc5000, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 101,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(2.5049)\n",
+ "tensor(0.4643)\n",
+ "tensor(6.6074)\n",
+ "tensor(0.1233)\n",
+ "tensor(12.2971)\n",
+ "tensor(0.1902)\n",
+ "tensor(4.0843)\n",
+ "tensor(0.1482)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "errs = []\n",
+ "errs1 = []\n",
+ "lens = []\n",
+ "fft_layers = []\n",
+ "for v in weights[\"1000\"].values():\n",
+ " flat = v.flatten()\n",
+ " lens.append(len(flat))\n",
+ " flat_fft = fft.rfft(flat)\n",
+ " topk = torch.topk(\n",
+ " flat_fft.abs(), round(0.2*len(flat_fft)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ " top10[topk.indices] = flat_fft[topk.indices]\n",
+ " reverse_top10 = fft.irfft(top10)\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " errs1.append(torch.norm(reverse_top10 - flat, 1))\n",
+ " print(err)\n",
+ " errs.append(err*err)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 102,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[800, 32, 51200, 64, 1605632, 512, 31744, 62]"
+ ]
+ },
+ "execution_count": 102,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lens"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 103,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "218.12065"
+ ]
+ },
+ "execution_count": 103,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 104,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "14.7689085"
+ ]
+ },
+ "execution_count": 104,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 105,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fft_conc = torch.cat(fft_layers)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 106,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(39.6843)"
+ ]
+ },
+ "execution_count": 106,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(fft_conc - conc5000,2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 107,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[tensor(57.1960),\n",
+ " tensor(2.2182),\n",
+ " tensor(1148.8655),\n",
+ " tensor(0.7960),\n",
+ " tensor(12109.6943),\n",
+ " tensor(3.3738),\n",
+ " tensor(578.7853),\n",
+ " tensor(0.9163)]"
+ ]
+ },
+ "execution_count": 107,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "errs1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 108,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "13901.846"
+ ]
+ },
+ "execution_count": 108,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 109,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(3.8017)\n",
+ "tensor(0.6525)\n",
+ "tensor(9.6627)\n",
+ "tensor(0.2314)\n",
+ "tensor(14.0121)\n",
+ "tensor(0.3602)\n",
+ "tensor(6.0212)\n",
+ "tensor(0.3187)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "errs = []\n",
+ "lens = []\n",
+ "errs1 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " lens.append(len(flat))\n",
+ " topk = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat))\n",
+ " top10[topk.indices] = flat[topk.indices]\n",
+ " err = torch.norm(top10 - flat, 2)\n",
+ " print(err)\n",
+ " errs.append(err*err)\n",
+ " errs1.append(torch.norm(top10 - flat, 1))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 110,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[800, 32, 51200, 64, 1605632, 512, 31744, 62]"
+ ]
+ },
+ "execution_count": 110,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lens"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 111,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "341.1233"
+ ]
+ },
+ "execution_count": 111,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 112,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "18.469522"
+ ]
+ },
+ "execution_count": 112,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 113,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[tensor(86.4496),\n",
+ " tensor(3.1551),\n",
+ " tensor(1531.9570),\n",
+ " tensor(1.5015),\n",
+ " tensor(14199.5361),\n",
+ " tensor(6.1879),\n",
+ " tensor(814.4448),\n",
+ " tensor(2.0511)]"
+ ]
+ },
+ "execution_count": 113,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "errs1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 114,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "16645.283"
+ ]
+ },
+ "execution_count": 114,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 115,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n"
+ ]
+ }
+ ],
+ "source": [
+ "flat5000 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " print(v.grad)\n",
+ " flat5000.append(v.flatten())\n",
+ "conc5000 = torch.cat(flat5000)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Wavelets <a class=\"anchor\" id=\"wt\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 116,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: PyWavelets in /home/jeffrey/anaconda3/envs/sacs39/lib/python3.9/site-packages (1.2.0)\n",
+ "Requirement already satisfied: numpy>=1.17.3 in /home/jeffrey/anaconda3/envs/sacs39/lib/python3.9/site-packages (from PyWavelets) (1.22.3)\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install PyWavelets"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 117,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pywt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 118,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#(cA, cD) = pywt.dwt(, 'db1')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# pywt.wavelist(kind='discrete', )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 143,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 18.427034\n",
+ "db1 18.427034\n",
+ "sym2 18.36348\n",
+ "coif1 18.393574\n",
+ "bior1.1 18.427034\n",
+ "rbio1.1 18.427034\n",
+ "dmey 18.671127\n",
+ "bior4.4 18.496372\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "for wavelet in wavelets:\n",
+ " errs = []\n",
+ " errs1 = []\n",
+ " lens = []\n",
+ " fft_layers = []\n",
+ " for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " #print(flat.shape)\n",
+ " lens.append(len(flat))\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " errs1.append(torch.norm(reverse_top10 - flat, 1))\n",
+ " #print(err)\n",
+ " errs.append(err*err)\n",
+ " # print(flat[0:10])\n",
+ " # print(reverse_top10[0:10])\n",
+ " print(wavelet, np.sqrt(np.sum(errs)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 156,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "bior1.1 15.07145881652832 16107.921875\n",
+ "bior1.3 15.25814437866211 16279.986328125\n",
+ "bior1.5 15.425777435302734 16436.12109375\n",
+ "bior2.2 15.66141128540039 16473.904296875\n",
+ "bior2.4 15.516386985778809 16417.8203125\n",
+ "bior2.6 15.544709205627441 16466.162109375\n",
+ "bior2.8 15.579778671264648 16503.138671875\n",
+ "bior3.1 25.344850540161133 24354.4453125\n",
+ "bior3.3 18.38104248046875 18552.30859375\n",
+ "bior3.5 17.479848861694336 17874.1015625\n",
+ "bior3.7 17.260520935058594 17713.20703125\n",
+ "bior3.9 17.19255828857422 17672.990234375\n",
+ "bior4.4 15.1978120803833 16241.7060546875\n",
+ "bior5.5 15.467646598815918 16527.41796875\n",
+ "bior6.8 15.204909324645996 16253.8125\n",
+ "coif1 15.142683029174805 16156.6630859375\n",
+ "coif2 15.175430297851562 16219.298828125\n",
+ "coif3 15.218149185180664 16275.017578125\n",
+ "coif4 15.248283386230469 16304.376953125\n",
+ "coif5 15.278726577758789 16337.1923828125\n",
+ "coif6 15.300649642944336 16357.455078125\n",
+ "coif7 15.324337005615234 16380.2119140625\n",
+ "coif8 15.34239387512207 16396.9453125\n",
+ "coif9 15.35299301147461 16408.88671875\n",
+ "coif10 15.358224868774414 16412.31640625\n",
+ "coif11 15.375347137451172 16429.083984375\n",
+ "coif12 15.383316993713379 16440.47265625\n",
+ "coif13 15.401575088500977 16450.142578125\n",
+ "coif14 15.413949012756348 16466.5859375\n",
+ "coif15 15.430389404296875 16478.208984375\n",
+ "coif16 15.438526153564453 16489.732421875\n",
+ "coif17 15.44447135925293 16493.5625\n",
+ "db1 15.07145881652832 16107.921875\n",
+ "db2 15.11799430847168 16146.3642578125\n",
+ "db3 15.206748008728027 16251.126953125\n",
+ "db4 15.276558876037598 16337.6650390625\n",
+ "db5 15.346190452575684 16396.228515625\n",
+ "db6 15.424012184143066 16471.95703125\n",
+ "db7 15.465736389160156 16520.9609375\n",
+ "db8 15.5084228515625 16558.216796875\n",
+ "db9 15.579204559326172 16622.1484375\n",
+ "db10 15.634806632995605 16672.583984375\n",
+ "db11 15.69124698638916 16721.88671875\n",
+ "db12 15.76386833190918 16791.78125\n",
+ "db13 15.807873725891113 16828.6328125\n",
+ "db14 15.84904956817627 16859.560546875\n",
+ "db15 15.879130363464355 16884.310546875\n",
+ "db16 15.916594505310059 16917.77734375\n",
+ "db17 15.97330093383789 16964.7890625\n",
+ "db18 16.010889053344727 17004.966796875\n",
+ "db19 16.06007957458496 17043.080078125\n",
+ "db20 16.109506607055664 17080.361328125\n",
+ "db21 16.15558433532715 17122.78125\n",
+ "db22 16.195322036743164 17152.8046875\n",
+ "db23 16.23825454711914 17190.244140625\n",
+ "db24 16.28815269470215 17229.99609375\n",
+ "db25 16.29660415649414 17237.244140625\n",
+ "db26 16.331958770751953 17263.62890625\n",
+ "db27 16.375545501708984 17302.498046875\n",
+ "db28 16.413320541381836 17331.599609375\n",
+ "db29 16.437959671020508 17352.27734375\n",
+ "db30 16.50661849975586 17411.228515625\n",
+ "db31 16.53733253479004 17433.791015625\n",
+ "db32 16.5701904296875 17458.037109375\n",
+ "db33 16.599777221679688 17484.1953125\n",
+ "db34 16.628063201904297 17505.951171875\n",
+ "db35 16.64190101623535 17514.66796875\n",
+ "db36 16.680456161499023 17541.33203125\n",
+ "db37 16.730104446411133 17587.6328125\n",
+ "db38 16.75263214111328 17598.830078125\n",
+ "dmey 15.428367614746094 16479.267578125\n",
+ "haar 15.07145881652832 16107.921875\n",
+ "rbio1.1 15.07145881652832 16107.921875\n",
+ "rbio1.3 15.1613130569458 16189.78125\n",
+ "rbio1.5 15.32840633392334 16338.216796875\n",
+ "rbio2.2 16.183170318603516 16984.47265625\n",
+ "rbio2.4 15.833732604980469 16793.46875\n",
+ "rbio2.6 15.841042518615723 16820.513671875\n",
+ "rbio2.8 15.870935440063477 16858.255859375\n",
+ "rbio3.1 112.47295379638672 58394.6875\n",
+ "rbio3.3 19.817306518554688 20010.50390625\n",
+ "rbio3.5 18.20765495300293 18729.5625\n",
+ "rbio3.7 17.90874671936035 18495.369140625\n",
+ "rbio3.9 17.855627059936523 18441.083984375\n",
+ "rbio4.4 15.365104675292969 16364.685546875\n",
+ "rbio5.5 15.447882652282715 16390.2890625\n",
+ "rbio6.8 15.320694923400879 16363.552734375\n",
+ "sym2 15.11799430847168 16146.3642578125\n",
+ "sym3 15.206748008728027 16251.126953125\n",
+ "sym4 15.159475326538086 16207.3779296875\n",
+ "sym5 15.204032897949219 16260.142578125\n",
+ "sym6 15.191091537475586 16246.744140625\n",
+ "sym7 15.236370086669922 16293.701171875\n",
+ "sym8 15.241791725158691 16298.5556640625\n",
+ "sym9 15.26644229888916 16327.4296875\n",
+ "sym10 15.264242172241211 16323.5576171875\n",
+ "sym11 15.332569122314453 16396.794921875\n",
+ "sym12 15.310770034790039 16371.40234375\n",
+ "sym13 15.304075241088867 16360.4248046875\n",
+ "sym14 15.317804336547852 16377.61328125\n",
+ "sym15 15.378673553466797 16436.265625\n",
+ "sym16 15.33505916595459 16392.92578125\n",
+ "sym17 15.344695091247559 16404.30859375\n",
+ "sym18 15.363746643066406 16418.08984375\n",
+ "sym19 15.430442810058594 16479.15625\n",
+ "sym20 15.377063751220703 16438.19140625\n",
+ "min: tensor(15.0715) bior1.1 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "#wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " #print(flat.shape)\n",
+ " lens.append(len(flat))\n",
+ " to_cat.append(flat)\n",
+ " flat = torch.cat(to_cat, dim=0)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 157,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " conc2.abs(), round(0.1*len(conc2)), dim=0, sorted=False\n",
+ " )\n",
+ "top10_og = torch.zeros(len(conc2))\n",
+ "top10_og[topk_og.indices] = conc2[topk_og.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 158,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(15.5541)"
+ ]
+ },
+ "execution_count": 158,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(top10_og - conc2, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 159,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "42.212055"
+ ]
+ },
+ "execution_count": 159,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Problem: weights with only a few parameters cannot be represented with the wavelets"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 126,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat5000 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat5000.append(v.flatten())\n",
+ "conc5000 = torch.cat(flat5000)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 127,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 127,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc5000, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 149,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "dmey tensor(15.4284)\n"
+ ]
+ }
+ ],
+ "source": [
+ "wavelet = 'dmey'\n",
+ "coeff = pywt.wavedec(conc5000.numpy(), wavelet, level = None)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 150,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACeNUlEQVR4nOydd5xcZb3/38/0ur2mbcmmN0pCL6EXQVBERBS8iljwil7vFQsXC6iIBX8oeEUuglcUkA4iSAudhCSEFFI22ewmm2yvU3b6+f3xnDNtZ2ZnWzblvF+vfe3OzJmZZ2Znzvf5ts9XKIqCjo6Ojs6Ri2GqF6Cjo6OjM7XohkBHR0fnCEc3BDo6OjpHOLoh0NHR0TnC0Q2Bjo6OzhGObgh0dHR0jnB0Q6Cjc4gjhFgphGid6nXoHLrohkBnShFCNAshhoQQXiFEhxDifiGEa6rXlQshRK0QQhFCmCbo8WxCiH4hxJkZbrtDCPHoRDyPjk42dEOgczBwsaIoLuAYYDlw02juLCSHzGc53YAoihIAHgauTjvOCFwJPHDgVqdzJHLIfHl0Dn8URdkH/BNYLIQoFkI8K4ToEkL0qX/P0I4VQqwSQvxECPEW4AfqhRD/JoTYKoTwCCGahBBfSjp+pRCiVQjxbSFEpxCiTQhxqRDiQiHEDiFErxDie0nHG4QQ3xFC7BJC9AghHhFClKg3v67+7lc9mRPV+3xeff4+IcQLQoiapMdThBDXCyEagcYML/8B4DIhhCPpuvOQ39F/5npt6ajP1ZB0+X4hxK1Jly8SQmxQvZC3hRBLk267UQixT32e7UKIs7I9j87hg24IdA4ahBAzgQuB95GfzT8BNcAsYAj4XdpdPgtcB7iBFqATuAgoAP4NuEMIcUzS8VWADZgO3Az8EfgMcCxwKvDfQog69dh/By4FTgemAX3AXeptp6m/ixRFcSmK8o4Q4hLge8DHgXLgDeBvaeu9FDgeWJj+2hVFeRtoU++f/Pr+qihKJI/XlhdCiKOB+4AvAaXAH4CnhRBWIcQ84GvACkVR3EhD1Dza59A5BFEURf/Rf6bsB3mi8QL9yJP53YA9w3FHAX1Jl1cBPx7hsZ8EblD/Xok0Jkb1shtQgOOTjl8HXKr+vRU4K+m2aiAMmIBa9b6mpNv/CXwh6bIB6anUqJcV4MwR1nsT8C/17wL1/kfn+dpak25TgIaky/cDt6p//x64Je2xtiMNXgPS4JwNmKf6s6H/HLgf3SPQORi4VFGUIkVRahRF+aqiKENCCIcQ4g9CiBYhxCAyHFOkxs019iY/iBDiAiHEu2qYpx/pXZQlHdKjKEpU/XtI/d2RdPsQoCWqa4An1PBJP9IwRIHKLK+hBvh/Scf3AgLpfWRcbwb+DzhDCDEN+ASwS1GU9/N8bflSA3xLW6f6WDOBaYqi7AS+AfwQ6BRCPKSuRecwRzcEOgcr3wLmIXfsBSTCMSLpmLh0rhDCCjwG/BKoVBSlCHgu7fjRsBe4QDVQ2o9NkXmMTJK9e4EvpR1vV2TIZ9h6M6EoSgsypPQZZFjogTG+Nj+QnGuoSlvnT9LW6VAU5W/qGv6qKMopSIOhAD/PtWadwwPdEOgcrLiRO/R+NUn7gxGOtwBWoAuICCEuAM4dx/P/D/ATLeErhChX8wCozxED6tOO/64QYpF6fKEQ4vIxPO8DyDj9ycCD6nWjfW0bgE8LIYxCiPORYR+NPwJfFkIcr1ZbOYUQHxFCuIUQ84QQZ6qGJ4B8/2NjeA06hxi6IdA5WPkNYAe6gXeB53MdrCiKB/g68Agysftp4OlxPP//U+//LyGER13D8epz+YGfAG+p4ZUTFEV5Arl7fkgNZW0GLhjD8z4GlAAvK4rSNsbXdgNwMTLvchUyn4D6WGuBLyIT733ATuBz6s1W4Dbke94OVADfHcNr0DnEEIqiD6bR0dHROZLRPQIdHR2dIxzdEOjo6Ogc4eiGQEdHR+cIRzcEOjo6Okc4E6KeeKApKytTamtrp3oZOjo6OocU69at61YUpTz9+kPSENTW1rJ27dqpXoaOjo7OIYUQoiXT9XpoSEdHR+cIRzcEOjo6Okc4uiHQ0dHROcLRDYGOjo7OEY5uCHR0dHSOcHRDoKOjo3OEoxsCHR0dnSOcCTEEQojz1UHXO4UQ38lwu1UI8bB6+2ohRK16/TlCiHVCiE3q7zMnYj06OjqTTNd22P36VK9CZ4IYtyFQRwfehdReXwhcKYRIH879BeS82QbgDhJTj7qBixVFWQJcgxzVp6Ojc7Cz6jZ4+t+nehU6E8REeATHATsVRWlSFCUEPARcknbMJahj94BHgbOEEEJRlPcVRdmvXr8FsKvTkXR0dA5m/D0Q9Ez1KnQmiIkwBNNJHcrdSurA7pRjFEWJAANAadoxlwHrFUUJZnoSIcR1Qoi1Qoi1XV1dE7BsHR2dMRPoh5B/qlehM0EcFMlidc7rz4EvZTtGUZR7FEVZrijK8vLyYZpJOjo6B5KhPogMQSw61SvRmQAmwhDsA2YmXZ6hXpfxGCGECSgEetTLM4AngKsVRdk1Aes5vAn54M+XQue2qV6JzpHMUL/8Hda9gsOBiTAE7wFzhBB1QggL8CmGD9Z+GpkMBvgE8IqiKIoQogj4B/AdRVHemoC1HP50N0LTq7B39VSvROdIJRqB4KD8Ww8PHTDe2tnNT5/bylBo4r2wcRsCNeb/NeAFYCvwiKIoW4QQPxZCfFQ97H+BUiHETuA/AK3E9GtAA3CzEGKD+lMx3jUd1vi65e+Qd2rXoXPkEhhI/K1/Dg8Ya3b38sc3mjAbxYQ/9oTMI1AU5TngubTrbk76OwBcnuF+twK3TsQajhj8qiEI6l9AnSki0J/4Ww8NHTC6vEFKnRZMxolP7R4UyWKdURD3CPTSPZ0pYqgv8XfIN3XrOMLo8gQpc01Odb1uCA41NI9A/wLqTBW6IZgSujxByt26IdAB8Kk9FHpoSGeq0CqGQDcEB5AuT5By3SPQAcDXI3/rSTqdqSLZI9BzBAcERVHo9k6eR3BIDq8/ookni/UcgcZLH3awbk8fTosRh8WE05r4PafCzcwSx1Qv8fAiJTSkb0gOBJ5ghGAkphsCHRUtNKR/AeP84Okt7OsfGnb9iYYtrLC3882bfoUQE19yd8QS6AdhBCWq9xEcILo8UnlnspLFuiE41IiHhvTYLIA3GGFf/xD/ee5crjttNv5QBH8oij8UwfKXX1I+sJmOgZ9QVWSf6qUePgz1gbsKBvfpn8MDhGYI9GSxDkSCibJRPVkMwM5O+T7MqXRjMRkocliYVmSnocjADO8m7CJEY1PjFK/yMGOoHxwlYLJDWDcEBwLdEOgk0HoIzA49NKSyo0MaxjkVrtQbWt7GEAsD0Nms6zJNKEN9YC8Gi0P3CA4QcUOgVw3pxPMDxbXSECjKlC7nYGBnpxeLyUBNqTP1hl2vxv/0tu04wKs6zBnqA1sRWJx6juAA0e0NYjYKCu3mSXl83RAcSmgVQ8W1oMT00j2kRzC73IXRkJYMbloFNacQxQB9u6dkbYcywUiUV7d1cvNTm3ltR9r8j0C/9AjMTt0zPUB0eYKUOq0Y0j/nE4SeLD6U0BLFxbXyd8gnd2VHMI0dXo6tKU690tMBnVvg7B/i7dhNsW8fA0PhSdtNHS4MBsK8uq2Tf33YwaptnfhUlcu2gQCnz1VngChKUmjIqW9GDhBdk9hDALohOLRIDg2B7CVw5SHW2tcCsQiUzp60pU0FPrVi6MrjZqbe0LRK/q4/g+jmF5nlb2Nr2yAn1KcPxdMBiMYUvvbX9by0tYNwVKHMZeGjR03j3IVV/M9ru+j2Jg0NDPshGgJ7kZ4jOIB0eYJUFtgm7fF1Q3Ao4e8GgwkKpsnL+brl/7wRfJ3wxVcmb21TgFYx1FDhTr2h6VVwlELVUmyVDdS2beCx/UeOIdjV5eXVbZ1ce2p9Xsfv7vbyz83tXLS0ms+dVMvRs4rjobanNuxj/Z7+xMGavIS9GCwu8PcNezydiafLE2TxtMJJe3w9R3Ao4esGRxlY1RNfviWkvk450OYwSy5rFUNzK5MqhhRFJorrTgeDAUflHIqEj6a9rVO0ygPP4+tbufUfW+OVJiOxvV1+jr58+myW15ak5FtKXdZUj0DrKrYX69VrB4hYTKHHF5rU0JBuCA4l/D3EnKX8X/vbDBhE3l/CplA/25WhVGmAw4DGTi8Wo4FZyRISXdvB2w71K+Xl4joABvcdOZVD/X5ZNru9PT8Zku0dHgwCGtJLcJGdrFqDHpCYRWArkqGhIy1H4OuBvWsO6FP2+UNEYzJkN1nohuBQwtfFVoeb2xv/xquO/HdjvzAHuKmsFPpbJnmBB5bGDg/15c7UQR1Natno7DPk7xJpCER/M4HwkTFoXTME29oH8zp+R7uH2lInNrNx2G2l6smnxxuSVyR7BBbXkVc++vad8KcLDuimqsurNZNNXo5ANwSHEr5umq3yw+AziLxDQ4PE2G02EettmszVHXB2dHiZW5mWH9j1KpTMhqJZ8rKaWJ+ptNPYcWSEMfqH5El7W54ewY5OD3Mqh3sDkGhg0k5GCUNQlAgNHWYhx5wM7JWFF0l9KpPNZHcVg24IDi38PTSru1+fMOTnEUTDeAUEDQY6uj6c5AUeOLSKoZSO4kgImt9MeAMAFicRRyU1ooMt+weGP9BhSJ8v/9BQIByludvHvHSDqqKJnCU8gn75WysfRYFIYLxLPnQYbJO/d750wJ5SNwQ6CSJBCA7SjIzV5u0RBD3yWGB33+ETJ0/WGIrT+p7UvtHyAyrG0nrqjJ1s2Z9fqORQZ2BIGoIdHR6isdy79V1dXmIKzK3KbAi00FB3skdgMMmwkNbDciSVkHqSDEEsdkCesturGwIdDVVnqCUmY7I+kyW/ucWBAfxC/ptbPIdP5UxcYyg5pNH0KggD1J6acqwoqaPe2MWHbUeGIejzhyhymAlGYrT05D5Ja+9jNo8gkSNQDUGgXyaKhZgwQ/BuUw/9/tC4HuOAoCjSELgqwdsBHZsOyNN2eYLYzAacluE5nIlCNwSHCv5uFKA51A+Az2TO6wuoBAbiHkFLsGcSF3hg2alWDNUkVww1rYLpx8r4dTIldZTGumlq6x5xhzzZRKIxVjdN3v8hGIniD0U5vq4EGDk8tKPDi9koqC3L3KFuNRlx20x0JyeL7Wont1l978dhCALhKJ+5dzV/fOMQyF8F+mUYbNmV8nLjvybneV7+MTz8mfhFbVbxZM7U0A3BoYKvmw6jkaGY/EL6jKa8QkNDQz3E1A9Qc2wIYodH5cyO9IqhoX7Ytw7qzxh+cIlsrCoLt9E8wg55snl2YxtX3PMu61p6J+XxB9SKoRW1JRjEyAnjHe0e6stcmI3ZTwXlyb0EqiHwhrx0o4ZG8iwhjWb47HV7g0RiCtvaDoGJe1p+oHopVB8FjZOUJ9i7Brb9I56Y7/JO3qxiDd0QHCr4umkxy0ZwgzDgMxjzShb7NelqoNlkksNEDgMaO72p+YHmN6QQX1p+AIj3EsiE8dSGhzbs7Qfgpa2dk/L4faohqCq0UVvqHNEj2N7hyZof0Ch1WZIMQT/Yi/j5ez/ny9v+V16Xx+ew1dPKigdXsKVnS8r1WiK0sfMQqOjS8gPuaphzDrSumZwyUn+v/Czvfh1IeASTiW4IDhX83TSbpWja7KLZ+AyGvOYW+wIyDDHLUsR+k5FQz6E/pMUXjNDal1YxtOtVqYY5Y8XwO6i9BHXGTj6cYkOgPf+r2ybHEGix9mKHhXlV7uG9BOFEhY/2Ps7LUjqqUeaypvYR2IvZ0rOFzpBahZVHL8Eezx7CsTDvd7yfcr1mCPb2+RkKHeTeqqdd/nZXw5xz5cl61yTItmgqw+pj64ZAJ4Gvm2aLBbvJRl1BHT6RX2exd0iGIBYWzUURgj0dH0z2SiedXV3ydc9NTxTXngKmDN2X9mKwFbLM0TulJaSxmMKW/QM4LEa2tXsyzlkeL5pHUGg3M6/KTUuvP9EV3PI2/Gw69OwCErvwOVkSxRplKaGhfiLWApoHmvFEhlAgrxyBV/2s7uzfmXK9lntQlMT/9aDFs1/+dlepuajiiQ8PKQr41RzSzlcIR6L0+cOTNqtYQzcEhwq+LpqtdmoKanFZXPiFkleOwK9KAiyadjwALb3bJ3OVB4QdHWknsL4W6G1K7R9IRggormOOuYsP9w+iTFEDVEuvH18oymdOqAHglUnwCgbUZrJip4X5VW4UhUQj3fsPymYodT7DjvbcFUMapS4Lff4w4XAYggPsNZsJx8JElAhBIfIaV+lTj2kaSE0KJ+shaRVMBy2edlVjyQ4GI8w+c+LLSIOD8n9UUg8De+hvld9X3SPQkfh7aDabqC2oxWl24lViee3EfKr7vrDiKACaB/dM5ioPCI0dntSKobjs9MrsdyqpozraRo8vRMdgfmJsE43mjXx02TRqSh2TEh7SPIIiu5l5VQWAWjkUCcLWZ+RBAbmO7R0ebGYDM5MrrzKg7Ub7e6UM+i4Rid/mFSKvz6FHLXXe2b8zxRB3eQO4bSbMRnHw5wkG22RYSGPOuVLQsX0CvWzNG1hyOQChHS8DkzeiUkM3BIcIIV8X+w0KNQU1OM1O/ESJ5REa8ql5hApHBWUYaQ50j3CPg5/GTm9qxVDTq/ILWj4/+52K63AF2jAS5cO2qQkPbd43iNkomFPp4ox5Fby1s3vC4+L9/jAWowGHxcisEgc2s0FWDu16BYLq61YNwY4OD3Mq3MOnu6WhiZ3FDUEsEdLyGgx55Qi8YflZ9YQ8dA8lPoNdniDVhTbqypw0pnsEmx+DptdGfOwDhifNEMw+S/6eyPCQX60mm74cimqwtEgpC90j0AFgT0AW69UW1uI0yZrvobBvRJ0Xn/oFdJqd1JhctEQPki7Qxpfgj2dCNDzqu+7o8KQqZbauhVknyhBQNkrqMSgRpolutuybmoTxlv0DzKlwYzUZOXN+BcFIjHeaJtYw9/tDFDrMCCEwGgRzK91s7xiUJ1Wr9BAIyNe/oyO7xlAymkfg7VMNQThhSGVjYx65qnDimOQ8gZYInVPhTvUINj0Kj34e3vrNiI99wPC00a4U893HN0mvxlUO046GnS9O3HNoVX6OUph9JkUd72IiohsCHUlLWH556wrqcKiNPD7BiDXcvoi83Wl2UmuvpNlI/nMMJpPGf8m6f60SI0/8IVnpEhebC3qlEFjFgtx3VCuHjiscmJISUkVR2LJ/kEXT5Mn4+PoSHBbjhOcJ+v1hih2JkZzzKt3s3t8N2/8Jiz4m5SECA/T7ZYhspPwAyJkEAP5B1RAEu3GZpQHxWOx59RF4Q16sRvk4yXmCbm+IMpeVOZUu9vSqlUMt78CTX5EHBA4SfahYFLwdvN9v529r9tDap3pFc86V0ib+CeoL0UJDjhKYfSbmiI+jxE49WawDRILsRu6cawpq4l9Cbx56Q77IEAKwm+zUFtTQZzQy0Lkl530OCN1q0to3uhNhXGNI8wi0ctiyubnvqPYSLHcPsGUKQkPtgwF6fSEWT5dTpqwmI6c0lPHK1s4JTV73+UMU2ROVU/OrC1gaeE/u2hdfJr2CwEA84T5SDwEkQkPBwV4iwG5/O0vLlwLgtdjyqxoKe5nhmkGBpSDuESiKIj0Cl/QIFAX2Nn4AD10p1WPrzzh4DIG3E5QYmwflJmzNbvXE33DOxJaRaobAWQZ1pxHDwFnWDzNKhE8kuiE4FFATxWUmBy6LC6dZhob8eSiQ+qJBnBgQQlBbthCAlo71k77kEelWT+C+0YVGGtMrhrpUIb3yebnv6K4Go5X51m729g7FhdkOFFo4SvMIAM6cX8H+gQDbJ7BaZmAoTFGSRzC/ys3FxrcJ2cpkea2tEIKDSdPdRjYELqsJq8lA2NfLXrOJcCzCsvJlAHhN1rzLR10WFw1FDTT1S4/AF4oyFI5S7rYyt9JFCYNM+8fVIIxw1d+lhPjBYgjUZrIdfvnde69Zi+UfA/aSiVMj9feA0SJF/exFNNvmc5ph8jWNdENwKODrosVsptZeCZAIDeUxpcwXDeEQsiO5pno5AM092yZxsXkQ9CQ6nL2j8wh2dHqkNk6pWunSvV2eOEpm576jwQAldcxAhqK2HmABus37BxACFlQnDMEZ8yuAiS0j7fOHKHYkeQQlgjMNG2gsO0uWPNoKVY/Ag8tqYlrhyMNOhBCUuazE/L00qU2NCUNgydsjcJld1BfVxyuHkuWVawoM/K/lV1iHOuHKh2T5pK0wns+YctQQZrtSQpnLyhrNEBiM0HAWNL44MWWk/h6ZH1DzXeuMR7Mg1jjpg3B0QzAWIiF46YcHbrfi66bZbKLGNR0gHhqS3cUjGAIljNMgv7wzShdiVBSaB6d4Ull3ors56ukY1V0bO7zUl7kSFUNd22X8P1MjWTrFdRQHpALrgc4TbNk/SF2ZE6fVFL+ussDG4ukFvDKBchP9/lSPoLT1FewixCqTqsiqGoLt7R7mVrryFjIrc1kQgQF22uWOeEn5EgC8RnN+OYJwwiMYDA3SE+hJGAKXGcvTX2KZYSf3ln8HZqrd4bYCiAZTuqGnDLWZrFsU85kTZtHU5Uv0QDScI7uB2zaM/3n8vdIQqLweW4yBGOx+Y/yPnQPdEIyFfevgzTvkLuAA0D+4l36jkboiuevVQkMjegSxGD4liiFq4r43d2M2WZihGGkZmhx5g7zpTsxF6O/aP6q7NqZP0+reAWUjhIU0SuowDeyh3GU54B3GW/YNsHha4bDrz5xfyfo9ffT5xi/DPBSKEozEKEryCNjyOL3GMv7lkU1s2ApQVI9gXh75AY1SlxVTsJ9dVjvTnNMosBRgN9nxGvPTvPKGVI+gUAoA7urfFe9WXrj5V7D1GR4v+zIP+45O3Mmmvl8HQ3jI004MA2WVMzh1TjkAazWvoOEsQExMeMjfLRPFKm/6awgaHJMjZZGEbgjGgpbQ8XUdkKdrVmOqtSWyTj5RNTSC3lDIi89gIBQ08rtXpTteY3LSHJnaqiGlawcRjLQqZfh62/K+nz8UYW/vEHMq1BNYNCw7istHSBRrlNRD2MdJVdEDqjnU5wuxfyCQkh/QOHN+BTEFXtsx/s+SNqIy7hEM9UPji2wvPYftnT4pwW0rJDY0QJ8/nHgf86DMZcESHmSX2cjspA3JaPoIXGbpEYAsIe3yBCnES+nGP8DRn2Hv3H9jT68/MVvaViR/B6c+PKQM7qeLQpbOKmXJ9EJsZkMiPOQsk7mCiZCl9veAowyQhr0vCPuKV8Culyd1JOiEGAIhxPlCiO1CiJ1CiO9kuN0qhHhYvX21EKI26bbvqtdvF0KcNxHrmXQ0USjv6MIaY6XZsxeAGjXZm+oR5IjPBgfxGQSRsIleX4jmHj81tnL2iBixKZSj9u3/kOZYJR1KMRFP/t7Jrk75WuMaQ71Nsh0/X49ArRw6rqCfnZ3eAzbMXgtDLcrgESydXkiZyzIheQJtRGW8fHTbPyAWxjfnowTC6pAaW1F8hz1aj8ASHaBZxOInc5fZJSvXRggNRWIRhiJDuCwuyuxluC1umvqb6PIEOdYodY9YegVzqtzEkjWH4n0PU+8R+Hv20RYrZtnMIiwmA0fPLE4kjEGGh1rXjr+MVMsRkJhM1l99CvTvkZ/3SWLchkAIYQTuAi4AFgJXCiEWph32BaBPUZQG4A7g5+p9FwKfAhYB5wN3q493cKN5BN4D5BEEOjEpCtMLZgJgM9owCqPMEeRyy4Me/MJANCprkNe19FFbUEPAIOjsmroS0kjHdnYq0/EaizAN5V81NGwqWZdagpq3RyANwXxrD5GYkqgFn2Q2q2GoTB6BwSBYOa+C13Z0EYmOL9moKY8WauWjmx+DohrK550IqFIT1gKMET8mInlVDGmUuawEzD5CAuqLZHjHbXHjRRm5YEHVGXKZZU6ioagh7hGcaG2SU+WmHR1fj1YifDCFhiL9++hUijl6ZhEAK+pK+HD/IJ6AWn1WcyKgQPs4KnyiEenFqYagU81BhGpXytubXh37Y4/ARHgExwE7FUVpUhQlBDwEXJJ2zCXAA+rfjwJnCZmlugR4SFGUoKIou4Gd6uMd3GhWf5Q18GOlJdjPDMWIWU36CiFwmB1qaCjHlzAwiNcgiMVkZci6lj5qS2V4aXfb2klfd0aiYdy+FjosM3GUVOMM9xLLc2pYY6ecplVTqk7T0noRRuoh0CicCcJAZVSGozoGD0wScsv+QaYX2Sl2Zk5onzm/goGhMOv39I/refrVkthipxl8PVKDafHHmVtVgNCG1Kgn15mOaLw/IB/KXBZ6LfLEpHkETrMTD7ERQ0OazpDLIg14fWE9uwZ20ekNcIxhJ1QsBKub2lInJoNIiM/ZDh6PwOzvoMdQSn25fA3H15UQU+R3Coh7m/Q1j/1JAv2AEjcEWjLaVTVX9lXsOrgNwXRgb9LlVvW6jMcoihIBBoDSPO8LgBDiOiHEWiHE2q6uid2J9/lCI852TSHuERyg0FDMT60htbPQaXaq4ypz5AiCg/gMBiIxO3MrXaxv6aOm6hgAWnq2TuaSsxLracJIFFPlfOzF1RTjobUnv5xFY4eHujJnYppW1w4omA7WPHe2JgsUzqRIrRxqGzhQhmAgozegceqcMkwGMe7wUJ/qERTZLbD1KVCisOjj2C3GxJAa1RAsKVVGNfqwzGmh3SJDaVrC121x4yMKsbCspMuC5hG4zfL/1FDUwEBwgE5vJ/OjjTBDljVbTAZqy5wJtdSDxSMID+GIDmIorI7rMh09qwiTQSTCQwXTZde2quw6JuLyEjJZ3KUNrS+wyea63a+PSZIlHw6ZZLGiKPcoirJcUZTl5eXlE/rYP3luK9fctyb/O2j/sAMQGorGouwhQq0p9UTiNDnxG805PYKQv4eIECg4uXBJNTs6PVgLFmKPxWgZaJ7klWdm306p1Fg1eylF5dMwCIXG5vzWsqPTk6qd3709f29Ao6QOh1eWz47GI7jn9V1sah39CckXjLC725cxP6Dhtpk5rq5k3Gqk/ZryqMMMmx+H0jlQJcs851W62d7hQVF32QtKRpd4LLcpNFsMlAtHvFjBZXbhUVQl0hxS1JpH4FSH3WuhJUPgfZyKN2WY0JwKV0JzSDMEU5wsDvbJyjZ32cz4dQ6LiUXTCxMdxkaT3LWPxyOIy0uoOQJPECGgxGmRktfBQVmxOAlMhCHYB8xMujxDvS7jMUIIE1AI9OR530ln874B9vcH8m/1T64amkgt8gy0+doICai1pRo/p8WJz2jMmSz2Dcl1Om0lrKgtQVHgg/YgtTHBbv/ovRl/KMIZv1zF85vzr/RJp2PXRgAWLFlORbX817e2jtzX4AmEpcaQVukSi8l+hJE6itMprsPQ10yRw0zbQH45gmAkyk+f28avXhz9LIetbYMoCiyent0jABke2t7hobUvv/m/mej3h7CZDdgCXdD8Jiz+eLwxaV6Vm+YeH/uGZDhoTsHoEuVlJj9NZjOVojh+ndPsxBdTd6g5Poea4FyyRwDgUDbLA5INQaWblh6fTOSbHbJZcIo9gt3NMqFdMb0u5frjaov5YO9AouiguA56x+ERJMtLID2CEodFesB1p8lcyiSFhybCELwHzBFC1AkhLMjk79NpxzwNXKP+/QngFUWedZ8GPqVWFdUBc4BRbM3HTyQao6nLRygawxOMjHwHSPzDYmE1rjd5NPfJ5qsa17SU650mJ94R5hZ7A3K3UuwqY9nMIgwC1rf0UWtw0BIZ/S6rscPL7m4fP39+uyxFHAOhjm10ilKmVZRjKagCoKu9dcT7rW3uQ1Fgea16IhpsldUqY/AIGOql3h2lfSC/uQTtagjpzcbuUdf7b96nJYqzewQAp82Vhn5109irTqTgnEWdO6DAoo/Hb9OG1LzWItdf6xqdIXApg+w2mymmLH6d2+LGFwsRhZx5As0QaDmCcns5LrMLq2UfQZNLei4qcypcxBRo6vJJI6Y2wE0l+/fIap26utTu9ePqSglFY2zUPMXi2vGFhtI8gpQRlY4SqXQ6Sf0E4zYEasz/a8ALwFbgEUVRtgghfiyE+Kh62P8CpUKIncB/AN9R77sFeAT4EHgeuF5RlANa19jc4yekVmvE57KOhL8HXFLuYbQSCaOluftDAGoLalKud5qd+A0iZx+Bxy8TWRWF5bisJuZXFbB+Tx81tjL2EyEUHd1JTSvr293t49mNo2sEA4jGFNzeJgac6s7KKU9+nu6RH+vd3T2YjYJjZqmGQGtKG61HUCLDEkvsvXmHhvb3y+MiMYV/fTg6tdQt+wcpdVqoLMitHjmtyA5A7zgay/r8YQrtZlnPXjIbKhLzGbRS0Wcb5c59mnV0z9PWv5OQQWCLVcWvy7exURtTqR0vhGC6s46AdYCBkmVS/kNFqxxq7NQSxlMvM9HfIYc5lVbXply/vEZ+FtfsVk/gJXXSaI1VDkIzBHY1R+AJpqqOzj4T9q2VlUUTzITkCBRFeU5RlLmKosxWFOUn6nU3K4rytPp3QFGUyxVFaVAU5ThFUZqS7vsT9X7zFEX550SsZzQkj8fr8eaxQwwH5Idekz2e5IRxc/9O3NEYpaoh8Ici3PDQ+ygxq5ShzvEF7PbK3eXMIvmBPbammPf39DPLPYOYEOzt25n1vplobWvjJet/cnXxZn77ys68q300tuzrp1bZj1Er93RJQ2AK9Iz43q9u6mXZjCLsFrW6WBOby7eHQEOt7phr7so7WbxfnS3sspp4duPowmJb9g+yaHrhiIlZp8WIySDiTWFjYWAoRLkdGRZqOCvltppSJzazgc09ch2O2OiaCnf2S880Fk4MZnFb5EnbJww5ewnioSFLIr9TYa6m3RIlVHlMyrG1ZQ6MBpGUMC6Yco8g1NdKSFgSDW4qxU4LcytdrGnWKodq5e+xhof8PVJsziyr/IYNrT/q0/DZJ0DNtUwkh0yyeLJINgTd+XgE6jB4ylVDMMndxc2De6gNhxHqSfP1Hd08tWE/Hf3IGu4cyeJen/yA1pVKV/OYmiK8wQguizwZNrePToW0tPk5GsR+vmV5nJ2dHv65eXS74w+2bsMthiivlxLG2IqIGcyUiQE+zCEC5w1G2LRvgOPrE633dG+X82OdZVnvlxH1y1pr6KDHFyQUGTnHo+USrlgxk7d3jWy0NIKRKDs6PDkrhjSEEBQ5zPFRk2Ohzx/mWLFdnpRnpxoCo0Ewp8KNFxsxxKgTsE2qPpV3KBGijM8kMBhy5whCXswGc3weAUBlyEif0cjgtNQ5ElaTkZpSR6pHMIXJ4h5vEHuwiyFbRcbBR8fVlbC+pU/2gIy3hNTfE68YUhSFbm+aISipl+NYjebM9x8HuiHo8OBUd5l5ueWa+xb3CCY5NORvpyYSibeda1IE/V6BnyhKji/goGokGsrkfY+dJT9kA2HVEIyyqeyo3ueIYKRwYBuXFTfx21caR+UVaBVD7ulqv6EQ4CijjNzDYta19BGNKRxflxDjokvVGBpFCSQAVhe4KqmKtqMo0OkZ2SvY1x+g1GnhsmNmEI0pPL8lyQD274H+vRnv19jhJRJTMmoMZaLQbmZgHIag3x/m6PB6MJil5HQa86vcKBgIGZ2j3mXv9LdRHYmwbyhh1BLih7k73DV5iWRmDsrn7ywdbsjnVrgTHoF1aj2CD1r7qRK9iOQRlUmsqC3BG4ywtc2T8AjGmidIkpfwBCMEI7FJn1WsccQbgu3tHlbUyRNkXjs9rXS0dLb8wk1iaMgf9tMRHqQ2HAZnGYqi8LpqCLoGIQoEc4SG/GpJX5VbbSIqsVPmsrKhr4zSSJSWpElRIxHt3sWi6Dbenv55cJbzX+4X2dbu4aWt+b3+UCRGpF2Vv06K6xvcFUy3+HJq/7zb1IPJIDi2JlGxQvf2/DuK0ymuoyQkE9T55AnaBoaYVmRnQbWb+nInz36QFB568quJaVppJBLFI3sEAEUOy5hDQ4qi0O8PMd/3Hsw6QRq8NLQ8QdQy+pNrU6Cb2aEIe7yJU4aW/B3JI/CEPPFjNearxrMtOjw5PqfSRXOPj2AkmiKJMRVs2DtAJX04SmdkvP049dyxprlXvufO8vGFhtKaySZ7RKXGEW0IgpEozT1+Fk8rxG0z0TMaj8BZDq6KSQ0N7fHIJFVtWH4hdnX52Nc/RG2pgwG/9GJ8EX9WMaqhqAxpJCfpjq0pYtV+QW0kSos//9COZ/WfiSmCgfmfghVfpKrzdU4r7uHOVxrzKrvd2NrPzFgrYZMrkWgHcJYzzeTJqQa6uqmHJTMKExLOvh75fxhtfkCjpA6XTz0R5ZEn2N8/RHWhDSEEFy2dxurdPQlPYmAvdH6Y8X5b9g/ispqYVeLIa1nFDnO8F2C0+EJRimN9VPobZVIxA8fXlWIxGjA5RleJE41F2R0epCYq8IVicpwkJE3Ky50j8IV9qR6BorBoYCu2mGDXwK5hx8+pdCcqh6Y4WbxhTx9Vhj5MRRn7XKkutDOj2M57Wj9Bcd04Q0OphmCyR1RqHNGGoKlLKjLOrXJT5rLGRZ5yoslLOEqlMZhEj6BZbfqqNbnAYIiHhb6ycjaKqh+Ua25xCGnYHKbEiejYmmKaewPMMthpDuf5BYvFsH74KG/GFlM9sx5WfAFMNn5Q9hqb9w2yavvIxvCdXT00GPYjytPCOc4KShigqduHPzS8fNcfirCxdYAT6pPCQpq0xGgrhjSK6zD52rESipeG5qKtPxCv6rloaTUxBZ7X8iPeTvkFziA2tmX/AAunFWAw5Be+KrRbxmwI+nwhTtEmWWUxBEtmFLLlx+dhdZWM6uS6z7uPIDFqFJnE1L4n2i7fO4Lm1TCPYGAvhdFeiqJF7OrPYAjUMaSNnV6ZLA55pA7PAUZRFJr27sdGCNxVWY87rq6E95p75YaouHYchqBX9wimgsS4PhelTkueOYJuQEiX1VUxqTmC5sFmAGZZpfv5+o4u6sudnLeoCmKqIcgyt3gwECYkIlgVA0ZDQsdPC6+Ui0J6iTAQzGNnuOcd7L5WHoueyuxyl0zQLvsU9fufYVFhiP/38shewdu7ephnbMNUkXbydpXjisgv0bb24aWw61r6iMQUjq9LShRrYnOj7SHQKKlDoFBv7hnREAwGwniCEaYVyZPg3Eo3cytdMjwU9CaMcE/qCS0aU9jall+iWKPIYY4Lx42WgaEwpxk3ErKWQNXSrMeZjYZR1+ZrM4ZrjDK0FDcEyR7BCH0EKR5B63sA2A2zUgbZa9SXOzEIKSkyld3FzT1+bEH1+50lRwBwXG0JPb4Qu7p8soR0oDWn5EZGtGpETV5CNwQHjh0dHkwGQX2ZixKnJb8+An8P2ItkS3laaMgf9vPg1gfjO/nx0jzYTLVixO4sJxCO8m5TD6fPLafIYaHMpZbuZdmN7e70EjFEsQtTyvWLphViMRqwReXOY08+08o++CtBg533bCclhNNOuB4RDfKzmavZsLefN3dmVxENhKNs37OPMqV3eFzfWY4hFqYAf8Y8weqmXowGwfLa5IqhHbLrtHDmsOPzomgWAEscA7SPkCNoU3sIqgvt8esuWjqN91p6UxvhelJLcXd3exkKR/NOFAMU2c34QtG8KpnS6fMFONWwCc/0U1Pq8jMySkOg7drrLFouTX5P7CY7RmHEa849t3hYaKh1LQEsOOyL6A300htI9aasJqmN1NjhnVJDsGFvH1VCLQ3NYQi0HON7zb1q5ZAiiwhGQ7q8hDeIySAosk98hVAmjnBD4KW2zInFZKDUZaXHl09oKJHZxyk9AiUa5bmm57j4yYu5bc1tfOu1bxGOjb36Q6N5oJmaSAwcZaze3UswEot3oDao1RZ+kbmZp6WzlyEDuIypOwqb2cji6QUMDsk4fXP3CJVDIT9seYq3racyrTwpPFM+F+acx5L9f6e2QPDbl7P3JKzf08esqHrSTN/FO+Xc3lqbP2Pl0LtNPSyeXogracQjXduhtGHkE142VEMw19Y3YrJY6yHQQkMAH1lajaLA6k1JuYGexpT7bdaG1Y8gLZGyLNXIDgyN/rMTbdtEmRgkXHvGyAdbCyAfT1Bl18AuqmNQqJYwax6BEEIOpzFZRtQaSg4NKa3vsTFWz3RXYlpZOnMqXezo9EzpTIINe/qZaVKftyC7Iagvc1Lmssg8wVgrh9LlJdRmsnzDiuPlCDcEHuapnYxlLhkaGrEc0tedmCnqqmCr2cDn/vlZbnzjRkptpVx/1PXs6NvBgx8+OK61KYpCy2ALtaEAOMt4fUcXFpOBE9QSyvmV8kvpzTK3eH97Oz6DAZdp+HDyY2uK2dFTKecXd23OvZBt/4CQh4eCJ8mwUDInfQ3h7+anDdtY276Bxzdl7kt4Z1cPc4xq93B6glf94B9TGh7WSzAUivJBaz8nJIeFQHoEY80PALiqwGih1tg9YrJ4/4BmCBLv4+xyFwuqC9iyQzV+wpDiEQwGwvz5nWYcFuPw9ywH2u5vYAyVQwWtrwFgnnPWCEeSSMDmqZO1q38X9aEwFpfqESSFUN0WN16jKatHoChKqkcQCULbRtbHGphdODv++OnMqXDT0uMnpOoTTUXCeEPrAEsL1ZBXDo9ACMGK2hJW7+6Nz7wYdZ4gXV7CG6TMnb9M+Hg5Yg3BUCjKnl5/fMhJqdNCTEloumdFTej0Bnr5Uc9qrphWRbOnhR+e+EP+9pG/8aWlX2LljJXc/cHd7PeOXoZBoyfQgzfspTbgA0cZr+3o4vi6knhn7dJquZPO1t7f2d2NTwhc5uEVK8fMKqYlUsn0SITmDF/CFD74G7GCmfzL30B9eVpHY+2pULWEiu4Hcdb+gZvXfZ4vPPlzfKFUz+rtXT2c4O6R5bbajknDJV/H4qIQ29oGU4azrN/TRziqpCaKg15ZqTPWiiGQnkThTKbRRedgMKfx398/hNEgqHCnGtSLllbj6VH/v9XL4jmCXl+Iq/64mo2tA9z+iaUJyew80EZMjqWprKLzLbbGZuEuz1zmmIKtEFByS5irRGNRdg/spiE4hMlZgttmSgxtRx1XaTRmzREMRYaIKtGER9C+GRENsiHWQF3xNFxmV1aPIBpT2BdQQyMH2CMIRqJs3T/IXLtX5gPN9pzHr6gtYV//EG3RAjDZR19Cmkln6ABVDMERbAh2dnpRFOIeQan6po/YS+Dv4R2L4KInLuLJnve5atDDM8d+n8vmXobRYEQIwXeP/y4AP1v9s/wVTdPYPSA/SLXhCH2GQnZ2ejl9bkKB9OiZsorBn2VucU9PN36DAacpgyGoKWavUk5NOEKLL4dkwmAbNL1KR92lKBiG726FIHrC9fzA6KHAZKFUHM2agb9w8p8v48F1a+RuMBjhg739LLF1yN4LY2rOQgsNNTj8BCMxdncndparm3owiCShOUiEYMbaQ6BRNIvyaAehaIzeHAnatv4AVQW2uA69xkVLqykXAygImHkC9OyiY8DPFX94h+0dHu65+lguWjoty6NmWZI6WWzUlUNBL9WDH/COOAqLKY+v9CgGvuzz7iMYDTI7FAZ7MWUua4pH4DK78IrsVUNxwTnNI1ATxe/HGqgosFFfVJ+5hFRVmd01aMp7rRPJh/sHCUVjTDf15/QGNBrUSqe9fYGxVQ7lEpw7ABxRhuCJxidY0yZPUNu1iiG1yaZUjc/mlJlQFP6Bj696N1HpqOTRU37Njb39FARS3eJprml8ddlXWdW6ilf2jE0tcF2H1B2fEwqzuU/uik5LMgTTC4sALVmc+vyKouDp78FrEDiT9F00KgtsFBeXMC1ioCXUTzjbsItNj4AS44OS8wGGewTAX4x+NtqsfC/q4rVr/sTn59xMzNjNzzZ+iQvuv4WH3msmElOYEdmbucrHUQoIZljkCSM5T/Du7l4WTy/EbUtKmI1VYyidolkUBOSOPlfl0P6BoZSwkEZNqZO5ziEGRIEMU0WGuP73z7K/f4j7/20FZ86vzPBoIyxJ9QhGXTnU/CZGJcIGy7H4w37u2nAX33j1GwxFsshsxwe+jBxu0Xbrs8NhsBVR5rLQneQRuCy55xYP0xlqfQ+/rZIOSih32ZhdODujR6BVDm3vVw3wAU4Wf7C3H4CSWE/O0lGNEvX80esLyvDQmHIEAuzFxGIKPb6Qbggmg2gsyn2b7+ML//oCVz13Fa/seQmLEWrUZp+4R5AjYfzAxnv4TnkxR9mreOCCB5hdrQpmZegluGrhVcwtnstP1/w0PqEpXxRF4eldT3Nc4RwqolFWdwiqC23x+moAgzBgVKzyS5i2G+sYDGKJ+vAZDBkNAcg8wawhJwFivLHvjUyLgA1/gxnH8cFQGWajYGZaY1TLYAu//eD3rHTM4sLd66B9E9886XL+dfkzNLiOZZ/h7/xq0zdw2Dqwe/dkNgRGEzhKKGYAi8kQzxMEwlE27OlPLRsF2UMgjHEV0TFTNAtrsAcbwdyGoD+QUjGUzHz3EO3RAt4fknmO0kALf7n2eE6aPUr9I21JDi1HMEqPYNfLBISVD0uCXPzkxfzPB//Dy3te5pfv/TLz8aOY/KXt1utVj6DUmdpvIz0CJWtoKF15lNb32O9eDECZ28Lsotn0BnrpC6QqdtrMRmpKnWzuVvJe68t7XuaiJy7i7zv+TjQ2PhHjDXv7qXBbMfs6oGBkz65UHfvZ4wslPILRRAP8PVI7y2Ckzx8iGlP00NBkYDQYefSjj3LziTfTH+znjcFf4Zx9B880PUU4Gk78IzN4BDElxi/f+yW/3PA7zvH5+Z+Gz1BgKZCxQ6Ml4+xis8HMzSfeTJe/i9+9/7tRrfX9zvfZ69nLJYVSk+f1fXD63PJhCpZWowO/wUDQl/olaery4hZ+/ELgtGYuXzy2phi3p4LiGDy765nhB7R9AF1bYdmnaOryMqvEkRLvjikxbn7rZixGC/991m8QZiesug0UhSpXOU984o98b/kt2Bxd2Ot+zwcWY/YEr7MCo6+L+VXueIfx+3v6CUVjqfkBkBVDJfVy7OR4KJJqrtNFd9YS0lhMoX0gkFIxlMw0k4cupZDr/iHX/KOTrRw9qzjjsfngspowGkR85GS+bNr9Ep+aNp0Ox18os5fx5wv+zOcWfY5HdjzCqr2rht9hNIagfxdVliJciiJDQ27L8GQxsazJYs0QuC1uOdGvv4Um60KcFiMOiyk+pCaTVzC73MWuniGwuEdca6unlf9+87/p9Hfy43d+zFXPXcWmrrEPkt/R4WVRtRPh7RidR+ANyRLSsH90PUbJXcWqoS3TPYLJwWq0cvncy3nm0mew930Om8nGzW/fzPmPn8/fGv+IybWNvYPtKXH9cDTMd9/4Lg98+ACfmnEmv+jsxupSPxhCqCWkmTtrl5Uv45PzPslft/2VD3syyxBk4qldT2E32TnbKuPne4KOlLCQhsviYlAY6elNrcNu6vbhxE/AYMBpy3xiOmZWMS3KNC70eFjV8hKDL/8QOrclDvjgIWnkFn2MXV2+YfmBv237G+s71/PtFd+momQOnP5fsO1Z2PIEICsprlx0Kc9e9gTVVifXVVXwtsjibTnLwNfNomkFfLh/EEVReLepByFgeW0JMSXGc03PccWzV/AJ/ya+UmTh5rdu5rfv/5aHtz3My3teZp93lIPt1BLSGkN31hLSbl+QUDSWMTQEYA10gascc0EVMbOLqvDIA3ZyIYSsG883R9Dl7+J7L3+dT7sitBkNzDNey98+8jeOrjiafz/635lfMp8fvP0DuofSejy0ksw8wi07+nYw26YaY3sRpU4rff5QPKnvNDvxKtGs5aMpOYJ9awHYLObGwx6zi2TlkNa0lszsCifN3X4UmztnGCscC3Pj6zcC8PhHH+e2U2+j09/JVc9dxQ/f/uEwb2MkFEVhb6+fhQUhOfc5jxyB1WTEZTUlPAIYXXgok87QAfQITCMfcvjhD8XobJ/Pfy79KMfM7+K+Tfdxz8Y/YJ8JD3fcz4uPlLCgdAELShawqXsTq9tWc8MxN/AF83QE9yfKR0Fq6ueQmfj6MV/n5T0v8+N3fsyDFz6Y0uWbcW1hPy80v8C5NefiGOonhhGvwcXJDcPDDcU2N4MGEwMDvSQ7r01dPlxm1SW3ZzYE86vcXGO4hC86iwiLl/jXhj/yiTfukPLaiz4Gm/4O8y4gYi2ipcfHOQsTMe+9nr38v/X/j1Omn8Ilsy+RV5747/Dh0/Dcf8pqIrXmfJprGvdXnsl1Ox/kaxvu4BeF0zhrVlqJo6sC9r/PwoUF/G3NXtoGAqze3cOCajdrO1/ndxt+x87+nTQUzmZ6cIguWxHb971JT6CHmJKoMppbPJczZp7BGbPOYGHJwtwzAIqlRzDf1kfbQIBAJEC7r53uoW7KHeVMc02LN5NNyxQaUhTwdnHCMRfzyjlnYLivIV5C6g/72di9kV39u/CGvPjCPrxhL96wF3/Yj0Bw+bzLOXX6qcPWWOgwj1y5BqzvWM83V30TT3CAa/sHeHXge8xfejIGIfd2FqOF2069jSuevYKb37qZu866K/Fcmq7+CLvsHX072NG3g49WqEqm9mLK3EMoCvT6Q1S4bbgtbsIoBEM+Mp22UgxB63tgMPF+ZBblamlkpaOSKmcVr+x5hU/N/1TKfWeXuwhFY4RNBVhyTAL87fu/ZWP3Rn55+i+Z4Z7BDPcMTp9xOr//4Pc8uPVBXmx5ka8f/XUun3d5/P3JRb9fdpM32NUijDRDEFNiGR+nxGmR3lxyCemsE0Z8PkBWI6pe6oHuKoYj1BBow7HnVxVw0rQ5nDTtJHxhHxf+/mEKizpZOt3Ltt5t3L//fgBuOfkWLm24FN5XewMcSXFrZwV4speJFlgK+PaKb/Pt17/NQ9sf4qoFV+Vc28t7XsYX9nFJwyWw5v8YFG6WzSyRk6fSKLS58BqM+DypX+jd3V5OdcgPU7YcgcloYN7MKp7xf4y6mbt4pnQJnyhfKXf0q34GKLDs0+ztGyIcVagvkzHemBLjh2//EIMw8IMTf5A4uRhNcOnd8IfTpDH45APx5yrra+VPXhNfnbaAb636FrecfAsXz7446T0sB28XC9Uu3PUtfWzofpeKWa/yjVW7qS2o5Ren/YJznTUY7j4BTroFln2KaCxKb6CXTn8nazvW8sqeV/jjpj/yh41/oNJRyRkzz2BJ+RIisQiBSIBQNEQgKn/7Ql7aKyvYan6dTt+bvPBgap7FKIwUWSqxz3TxfPt7dIg5lNvLKbYVU2QtoliYKYwMYS6sJKD4WVNUzrr+7ax77jNs6d5CRBvqjvREnWYnTrMTl9lFT6CH61++nqVlS/nqUV/lpGknxd/HYodlRCnqR3c8yk9W/4Tprun8KVZOXXQr/29oBqc6UsNls4tm863l3+Knq3/KQ9sf4sr5V8ob8qwaemT7I1gMFi6xqtsMWxHl6ojLbo80BFrs3xMLYY1FIW2jow2ud1lUQ1C1hP0emFclT3JCCC6bcxl3bbiLPYN7mFUwK7F+1Qv1G5xYsngvb+17iz9t/hOXz72c82rPi1/vsrj4rxX/xccaPsZP1/yUW1ffyhv73uBnp/4sZUBOJvb0ynxHrUV9TrWZbHP3Zn6z/jesaVtDobWQUlspJfYSSmzyRxQNscc3j3DBxzAjRldC6uuWoyhJNOzphmCS2dGuaQwlPhBOs5Nq2wKi/vncespJAPETR4FF/eKkdf8BcjfbtiHn851fez5P7nySu96/iwvqLqDEVpL12Kd2PcV013SOrTyW0OCvaI+6U8pGk3GYHXQZDAT9qV+Spm4fH7EG4sdkY3lNMXet2sUNJ13IHzffxb6Vv2D68dfB4H7o2AINZ7Nrq4xzzlYT1Y/ueJQ17Wv4wYk/oMqZFjutWACn3wiv3CINyqKPyeu7tlNYNpd7zr2HG165ge+9+T18YV9iB+gsZyDioz+8FkvZy9yy/h7M05sQxipuPeFWPlL/EUwGk/Q4IJ50NhqMlDvKKXeUs6hsEdcsuobeQC+vt77Oq3te5cmdT/LQ9oeGvW6jMOIwOai02imLKPQHl/HFE4+lyllFqb2ULn8XLYMtrGraSqdxN2+0Pc8Lex8b/gbWzcLV9H/4d91HTIlhsiosVhSuWXQNy6uWs6BkAQWWAsxpg0TC0TBP7XqKezbew5df+jLLypdx/VHXc0L1CRTZzVlzFuFYmNvX3M5D2x/i5Oknc/vJP6PgN8sIzbuYWKfIuFn41LxP8UbrG/xq7a84ruo4GYoxmsGceyaBP+zn2aZnOa/2PIr8AXm8yTKsqCIxk8BAWciXMDIqWqGEw2CFfevhqE/TtSbIKUke7sfnfJz/+eB/eHTHo/zH8v+IXz9brVIbVBwUZVhrl7+L7735PRqKGvj2im9nfB0NxQ3877n/y9+2/Y1fvPcLrnruKu48405qC2uzvvYW1RBUG/oBaCbKnav+gxdbXqTEVsLVC68mEA3QG+ilZ6iH7b3b6Qn04LF66AVO+ftvWT59Bsd3vMUJfTuYUzRnmOcXjoUJRoKEY2Ei0TDhYD8Ri5VIfxONfTuw2Xyp3fSTzJFpCDq82M1GZhSnuvylLitbk7pbLUYLFmPSLsvfI+PmyUqKrgppzWOxrJIHQghuPO5GLnvqMu5cfyc/POmHGY/b793PmrY1fGXZVzAIA57ednoVd8b8AMgvYcBoQIS8cvKY1UQoEmNvrx/LzED8mGyctaCSO1/ZiTO0AoBndz3Ll5Z9SVZJqJUSTd1ypzy7zEW7r51fr/s1x1cfz2VzLsv8oCd/Qw5P/4caInKUQncjHPNZnGYnd519F/+56j/5yeqfsK13G/6Iny373mFPzQx46z+wlgk8wQrC/R/j0etvpNyVVLKqqY7mEJsrsZVwacOlXNpwaTzcYzPZsBgt2Izyt8mgfuz/72Ps627jfN/H+OLS84Y91sD+D9m6bw/v/Ohc+oJ99AR66A/00xfso3//evrW3E3fwosprFjMsX4fS1/8CfYLHoHKhVnXB2A2mvnE3E9wyexLeGLnE9yz8R6ue/E6jqk4Bp9lET2hSiKxExPrBPoCfXzrtW/xXvt7fG7R5/jGMd/A2PoeBAcYmHYqIL2JdIQQ/PjkH3PZ05dx4+s38teP/FV+pkcYAfnc7ufwhX18ct4n4e17ZEULCVlkbdeq7a7jUtRphsAT8uA0OzH27oKQl3D1MQwGIinyyhWOCs6YeQZP7nySrx39tfh3rshhodRpoSdqY1YgdfhPTInx3Te/iz/s577z7sOWoYM++T349IJPM6d4Dt9a9S0+/Y9P8/PTfs6pM07NePxe1RDEwq38qKyEJ16+DovRwleWfYVrFl2TqIBK44ZH3uKt1tVcfJKf1dse4/VgCzx9GcXWYmwmG8FokEAkQDAaJJo+mn1GBXT+C576FwDmWsHV/3yGs2adxVk1ZzHTPUZdrTw5sgzB7tfBaGFHh2BOpWuYjkfZSMJzflVeItm6uyplQmmoN+fYxPrCeq5ccCV/+fAvfHLeJ1lYOvxk8cyuZ1BQ4mGTqLcLj3Emx0/PXPkjB9iDkwBb2wZZUVvCnl4/MQWMhmD8mGwsnVFIdaGNdxsVllcu59mmZ7lu6XUpu5ddnT7KXBYKHWZ+tOp2IrFIakgonXiI6HR47r/g3FtkIlE9eVuNVn59xq+5+a2beazxMSodlSxxzeDjO3ez5KzbuG9THc9v62dBdUGqEQDZQ1AwI+PQlUzYTLacOz+KZlGy5308wUjckCbTNjBEdZENg8FAqb2UUntSbsg3BP2DsPQrULUE9m8A5VaZJxjBEGiYjWY+Oe+TXNpwKY81PsYDWx5gX3g9lMNJf7uLJWVLWFa+jNlFs/nt+7+ly9/FT0/5aSKstusVEAbay04ENlPszCxQVmYv48cn/ZivvfI1fvv+b/nW8m/lFJ5TFIVHtj/CnOI5LCtfJoel24uARJlkt0d+T+KhoSxTyuLKo6oIW59tFtA/LOxx+bzLeWnPS7zU8hIX1l8Yv352uYuOQRsoqV7vfZvvY3Xban500o/iCeeRWFG1gocueoivv/J1rn/5er5x7Df4t0X/lvJZbve1807HixTOXM8lHWuIuJx8ct4nuW7pdZTZc5cFV7lKGOxZwPePPx/R0Ubbzn/x7kdvZ33nemJKDJvRhtVklb+NVqxGK2ajGfPQAKZXbsV8zDWY6k7jf1btpjfUSjC6k1+t+xW/Wvcr5hXP46yaszhr1lkZPYzxcmQZguf+C4rr2N5xbcZwS4nTysBQmFAklrlDM0kvPI5TfRxvx4jzc7+87Mv8o+kf3LbmNh44/4GUf6aiKDy16ylWVK1ghnsGsZiCLdSHrWjFsK5WDYfZwZBQcDHEa60DrKgtoalL7uBjqKGhDJ3FGkIIzltUxd/W7OFHKy7kJ2t+zObuzSwpXxI/pqnbS32Zizda34gn3UbcnVQukiGiV29NlCom7eLNBjM/O/Vn3HTCTfJE0roO1j0H1jLWTa/i+U0Z+gdgfFPJMlE0C3u4DzsB2gcC8e5QjX39AaZnKR2Nq86qndGUyjLIdPG5fLAYLVw5/0qunH8lP3vhHf533at86kzY2P0B922+j6gSpcJewf3n35/yv6HlbaheRk9UrrHQnr2k9vSZp3PFvCu4f8v9nDbjNFbkMASbuzeztXcrNx1/k/yMDvXFPQK31YTFZKDbl+oRZGpshCTlUbWgokspJJMhOKH6BGa4ZvDIjkdSDUGFk30dZmBAJuiFYEPnBn73/u+4oPYCPtbwsdxvbhrTXNP48wV/5ua3b+aOdXewrXcbJ1afyNqOtazrWBevPjM4bZwtnHw1aGDm8d/L67FLnBZC0RjeYAR3cR3Vnk4+VnMuH5szwhpb14HXBzNWQt153PfC20w3HM1DF59Iq6eVl/e8zCt7XuH3G37P7zf8nlc/+WrqpmQCOKLKR6lYSLTjQ7o8wbi0RDLabidrHXdSiVccVSsnn5rhAksBNxxzA+93vs9zu59LuS3eO6BW4Wxt7aYAH2WV2ZtZnCYnIRRshiCb1fp7TaIhpMiu0vQRgemcu6iSYCSGJXg0VqOVZ5pSewp2dfmoLTfzk9U/oa6wjs8t+tyIrxOAU74hdfHX/UleztBDEPdWNAPq62TpDGk4Tpqd9j7HYjLENN6O4mSSegkylZC2qZPJMuLtlGJz2tqtLlldkjaXYBjhAGx5Mmuz0YyCaiKDy/jS4m/x8EUP8/aVb/PnC/7M45c8nmoEFAU6NkPV0ni5abEjt2Txt5Z/i5numdz81s34La6s5aOP7HgEu8nOR+o/Iq8Y6ot7BEIIypyWuEcw0gD7uPKoagjaozJ0lG4IDMLA5fMuZ13HupSegtnlLjpCVul1h3z4w36++8Z3qXJW8d8n/veYdsYOs4NfnPYLbjjmBp7f/Tw3v30zb+57kwUlC7hxxY04u/6LM2x/4GdeRhWSSXQXJ5eQNo98R79a3queW3zBKE6L3KPPcM/gmkXX8MAFD/DKJ1/h1yt/PeFGAI40Q1C5EONAC06G4mJzyZTlaCoDUpVHNbSxi3k2j1zacCkLSxfy63W/xp/Ulq/1DpxTcw4AO1vknICq6uwiYtpJ3mAMxufjNnXJUE4gpoaGTNlDQyCHahQ7zLy+zcPKmSt5fvfzcQntPl+IXl+IbuNz7PPu46bjbxqW+MyK0QyX/h4MJukVODPnOeQLUY2pr4uTZpfy4LXHp5SrysXsljHoPMMueaEaghmia5gKaSgSo8sbzNpMhrdDfhaSq2RKG6SxysX6P8Pfr8laYFDo0PSG1OlyZgdHVxxNYXpjoKdNnqArF8ePLcqQI0jGbrLzo5N+RKu3ld8aBjN6BAPBAZ7f/Twfqf9IYhMR6E+UnCIbnYYni0XGXgJvyKsagk6wFdKhfuQzjWC8tOFSTAYTf9/x9/h19eVOPKhebXCQX6/7Nfu8+/jJKT8ZsfonF0IIrl1yLU9e8iRPXfoUqz65ijvOuINPzv00HT2lzCp1yfc4j2YyjZTuYq2ENJ/KobjOkPSCfaFIYixrEmX2Ms6uOTvv9YyGI8sQVCwCYK5ojQ/yTmZEmYlMHoF2gsvQXZwJgzDw3eO+S6e/k3s33SsfNrl3QK3y8fXKHZS7JPsHUQv7RESInZ0e/KEIu7t91JU58UVHzhGALCM9e0ElL2/t5Pyaj9AX7OOtfW8BMixksHSyfvAJLq6/mOOqj8vrNcapWgwf+bXsMci1czPbZfeotwshBCc3lA3f6bV9IH9XLxvdGnKhNpXNyOARdAwGUJQsPQQgQ0OuNGNV2jBsQM0wGl+Qv7MYDE2KesSmsg51jkTlorhaaaaqoXRWVK3gU/M+xYORDt6PDhcrfGbXMwSiAT4595OJK5NCQ0DKWFenRcsRZPYI4jkCbwe4KuP3006ayZTYSjin5hye3vl0XCdpdrmLQUU+x9t7X+fh7Q9z9cKrObby2BFfaz7UF9VTX1gf/7zt65d9EnWFRpn3c+cvHFjilOePeHcx5OkRaIZAepe+YGZDMJkcWYZA3U0us+yjqmC4y6+5dhk9gmhE7ozS8wC2QjBaR9VOflTFUXyk/iM8sOUB9nr2pvYOqAQHpSGwFGQXL9NO8kMCrEqQrW0emrq9NJTa8CpRzBjy2sGfv7gKTzCCGJpHia2EZ1TJiZ0dXqxVT2Iz2WWCcSwce43sOh4JZ1luY9r2gZSxLl8wtnVkwlUBJhtzzN20DaQKtO3LMJAmBW/ncC+nbI48eWSYXwxIPZ7dqq5TFoOREJ4byRCocyQqF9LvD1FgM2XNJaXzzWO/yTSjg5vdZgLhxOtWFIVHdjzC0rKlLChV3+fwEEQC8dAQSIFG7TtiNpixGSxZx1UmDEEnuCrp8gQpcpixmjI3Vn5y7ifxhD280CwN5oxiB36Dk0GD4OZNd1NfWM+/H/PvmV/Y7tfh9vpxjY9t6ZHGrN6u9pWMxiPQQkP+kDSc1sL8uov9qkS7Va3ACkZwWXM3nk40R5YhKJzFkLCzwtGeMbZY5kwtjUthSG1TT/cIhBjT7OJvHvNNjAYjv1r7q5TeAY2YaghyhVQ0Q+A3CFwEeGdXN93eEHOLFXmdMXeoQOPkhjKcFiMvbe3h/NrzWbV3FYOhQV7e9zwmZxPfOOaGSYlLppA29nMY7Rtln8J4NYaSEQIKZ1Jv7qV9IPV/rhmG6izyEvLEVpF6XTxhnMUr2P06RIOAyOoRaCWgI3YXd2yRFVT2YvqHwokRonngMDv4YcVpNJtN3L3+zvj1azvWsntgtywZ1Rjql7+TPQK3lR5vKC7FkkuK2hvyqjpDHXFDkEs64djKY6kvrOfv22V4yGgQuAtL+XlJMd3BAX56yk+xGrPcf996eVLd/Xqe78RwtNLRfCaTpZOSIxACSmrz9wjUasRoTCEQjuGw6B7BpKEIQaMyg/mGvRlvL7CbMBlE5iH28YROhmoWZ3neoSGNSmclX1zyRV7e8zKr21ZzyexLUtrWnb49xDDEwxeZ0AyBz2BgujPK0x/IDufZBTF8BgOObF+YNGxmIyvnV/Dih+1cWHcRoViIx3c8znuDD2CO1PDJeZeP6rWNCbW7OCOKIj2C6uxD2cdM0Symiy7aB1M9gv0jyUv4chiCbHmCxn/JxqzaU7Iai8J8pag7tsjqLOQgm9HOtj2hZAGfGPTwwLa/srFrIyA7id0Wd0qHbnwDlGQIStXqmMEh2T2dTYo6HAsTiAbk51TzCLy5dfaFEFw+93I2dm9ka89WuYTSLp52u/hi1SksKluU/UV52uXv5jfzfRuGsafXj9VkoCiift/z0BnScFiMWE2GxPmjuDa/HIEvEXL2hdT3VA8NTR5d3iCbIzOYHtqdsWpDCEGpK0svQdrgiBRcldlPYjm4etHVzHDJZHCK5AJQHmim11wN5uyNMpoh8ArBknITOzrkjqzWFcUnBE5j7qlKyZy/qIpub4igbxq1BbXcsf4OIoqHhebP56XPMm6c5dk9gsH98v2vPmrin7doFhXRjmEewf7+IYod5vhEuBSCgzJc4kwzBEU10sXPdJJXFGh8EepXQuViWV2U4TPoVhVIc4aGIiE5rlM1BAP+0IiJ4mHYCvmP3n7KbcXc/NbNtHnbeGnPS1wy+5LU5ixN4ycpWaydyLUSUpfFrYaGUnMEPvWy22CR3oKrIj6LNxcXz74Yq9HK33f8nd5ALx+YnmVBMMTnC5fkvF9c6qXl7dzH5aClx8+sEgcGr2pURmEIhBApYTOK62T/xEiS2P6eRKI4KA2BniOYRBo7vGxXZmEL92cViit1Zhlin5bQSWEE4blsWI1WfnH6L7jp+JuY4U5UBymKwvRoK33O2pz3T4SGDCwqlaEug4Aqa0jOKx4hUZzMynnlWIwGXtjSwcWzLyamxIj0ncSyiazSyYWrQr7H0cjw27REcdXkeASu6AB+7wChSELAri2H/HTc6Kcni40mWS2SqZegaxsM7IE558hJbWGfrEpJQwgpFdGfa25x9w6IRVI9ghFKR4dhLcStKPxw3mfZNbCLz7/weSKxSGpYCLJ4BKoh8CQZggxziz1hmYx2aidCNVk8koZOobWQ82vP5x9N/+Dmt24mxBC3dvXg7xtBRVTzCLq3j2ljBtIjmFXikEbFaE153flQ7LTI4TQgPwuxMAyOoIybVITiC8r3yqnnCCaP7e0edijqCVerukij1GXJPKXMl1rrm4KzQoaOxjAMY3HZYq6Yf0XKdf3eAHW0MVSYu2MyOTQ0p0gagpklDixhLz6DwDEKQ+C2mTllThkvbGnn8jmX8/H6zzLUdS71ZfkPXx8XznJASRjcZNo3AkJWIU00xYlegk5PonJof/9Q1oE08TCgK0P+prQhcy9Bo5QOkIYgdy6hyDGCFHW8Yki+H/3+UEZ5iZyojX6nOGdxyexLaPW2clzVcdQV1qUeFzcERfGrtKHq2lwCl9mlzi1O8wjUclJ3RL6WgLUMfyial5jaJ+d9En/Ez2utr3F5/bXUhGCgrzv3nQbboET9zrS8NeJzpKPJT88scUij4q7KXe2WgRKnJTU0BCOHh/w98SKUuEeg5wgmj8ZODx029YPSmXk+QKnTksUjUCtBMuUIXJWgxDKfxDR8PYkv1Qj0tu3EKsLESubkPC5hCAS1brmbrStzQtCDT2SfTpaN8xZV0to3xL5eAyeXXA0xa1xsbtKJl+Fm2Mm1fSArciz5G7a8UXsJZorOlBLS/f2ZR1QCCe8vPTQECUOQvilofFGeuAtnyNcCOUtIc04p69gsNa9KG4hEYwwGInmVjqaQNJzmv1b8F8dXH88Xl35x+HGZksVpekMyR2AYliOIK4+G5PvaI4qA/HT2l5Qt4ZiKY1hRtYKvHXstg9gZ8uT4/iiK9LDmXQBmx5gMQa8vhC8UpabUAQOtUDB91I8hzx9JoSHInTCOReV5Ie4RTE1o6IiSmNje7qG8chp4KqEjiyFwWWUdcDr+HjnQw5ThQ6ztDDNVkmg8dKX88l3198y3JzG0TybJzJW5u2hNBhNWgwWfMFBiDlNb6uDomcUQ/ACfIft0smycvaASg9jEv7a0Y1d3JJnmFE8K8aayDEn3to1Qc+LkPG+8lyDRVOYNRhgMREYfGgJpCKJBeSJRvQ0CA7DnHThJLXt0TwOTPWsXcpHDkuKdDKNjC5TPB6OJQfWkM1JX8TDihqCfQmsh9557b+bjhvrkaFBrQkyu2GHBIBLzvWXVEMM8Am06mSsor+9UCoHOvDwCIQR/PPePmAwmDMJAi3AR8uUwBP5eGYYpnAkzj4Pm0RsCTXV0VolDht/mf2TUj1HitCY8goLpsqEyVwnpUD+gxA2BN6gniyedo2cVy47VioXQmT005AtFGQql7eiSEjrD0E4I2SqHAgNSi731vbzmmEbVAe3u6SPH551mBz6DQIS8PP+N0/jamQ0QGJDzipMSfPlQ6rJyXF0Jz29pp6nLS7nbSoFtlCeYsRL3CNLcf18PDLZOTn5AfV7FZGOG6I7PLm5Tewiyy0t0SHmJTJ8HbbefnCfY9aqM6c85V142GHI2n404paxjSzws1JdnV/Ew4jMJRphSFuiXRiMpRGI0CEqclhSPwCcgGkwtH40PpQl4QBhpC8sGyJGSxRoWoyVeqBA2u1GGcsxP0BLFBdVQc4r8fmfr58iCVjpaZx+S3/cxyJmUuiz4Q1EC4ajMGRXNyh0aSpOX8KvnHYeeI5g8/vuihVx7ar1MsnVtzxjT15pChoWHNOXRTGghgmy9BHvXyNDRUJ+sgBkBc18j3UoBZZUjVyw4zW4p+BX0YjMbMRoEscAAfoMBp7VgxPunc96iKnZ0eHm9sSuuB39AcCZ5Vcm0T0JHcTJCQNEsaoxdcUOgNZNlF5xTm8kyTZuLx/+TdvuNL8qT6YykzuzS2VkF6opyDafxdYO3PZ4o1gzGqJPFJpsML400tzitq1ij1GmlRzMEqsyEP5LmEWiGYKgfnOV0++Rud0wDV2wFGMODKWNkU/AkVfnUniz/3vPOqJ5iT480BNOjanl5+fxRLzOllwBkeChXaChNXkL3CA4kFQtl+V9v07CbtIqIYSWkmeQlNFxZTmIayfFKrSM0B67BJpqZnldTidPixGc0QSghFzCkzmgdSWcoE+ctkp2UHYNB6ssPUH4A1A5ty3CvKl4xNELp4DgQRbOoNfbEB8JoIaLqXKGhTPkBkAbCWpiI/8disPNFmH2W3CFqlDZAX4ssBU2jyGHGE4wQjsaG3ZYsLQHkrTM0DCFySlHHyWIIytyJoor4TIJw5mSxy9cbLx01iMTJcjQYHUU4Y774YPdhaBssdxVMP1YaulGGh1p6/VQWWLH2qf+7MSjdDjcEtblDQxnkJUAvHz0waCWRGSqHNA2UYU1l/t7MpaOg5g5s2UtIW95OSCO0bxxxeSVDzbSZszeSJeMwOaQhSHLLveqXO9d0smxMK7LHFUDTB9ZPKkKovQRpoaG2D6R7nS0sNxEUzaKahEewv38Ig4DKbDtXb0fmiiGQr6N0diLs0/6BPF4LC2mUzZGKmhl2i9ruPmPCeJghyE95NCO2wpEH2CfNIkim1JmkN6T1s2RIFpsNZqyqLlOXJ0ipy5q3FEbKUl3FuIWfpq7hekbyyVSPwFUl83gzVkDL6BrL4qWjXTtkwrkgu+BjNhIRBfX8UVInjW22QpG0/iTNEDjMemho8imfL2O8GSqH0isi4vi6s5+MhJA7xEwVLyG/bH2fe56sUGkfwSPw9eCKDdJrr8nnleA0O/EZUkv3fCFpCHJNJ8uF5hUcsESxhrN8uFfVtnHy8gMaRTUUxAYZGJBf1v39ASoLbJiMWb4emQTnkimbkzAEjS8CAhrSVCNzlJAW5hKe69gi3yc1uT7mHAHIDcxIHoGvG+zDP/fFSSWubrPqEURTu7MT8hKd4B5ZXiIXrqJSCvCzq2u4jAUgK4YcZQkJkpqToH3TyK8vib29fmaVOGUfQtmcrBMHc5HwCNTzx0glpMOUR6M4LMZhQ7Mmm3EZAiFEiRDiRSFEo/o7Y/eFEOIa9ZhGIcQ16nUOIcQ/hBDbhBBbhBC3jWcto8Jsh5L6nB5BT7JHEPJDZCh7aAjUprIMoaF9a2U1Q83JMrzRvin32rplothfkN/UJZfZhd+QqvPiD6qNPKPoI0jmihUzufrEmszDYSaTdKmOwCD07pqcjuJk1Mohq6eVWExRS0ezhIUURR1ClENWu7QBBvZKwbbGf8H0Y4Z7EKXq/zeDIdBO6gOZmso6Nse9AXlMGIOQHcmjZqTQUCQkE/XaySxtjYOBMNGYklAgjaau1xv2ys+gLyEvUTbGgezOglLsIsTu9v7MB3jaUnWBak6Webk97+b1+IFwlPbBQMIjGOPci2GhZa2ENEMYGpDFEGanPCcxNcqjMH6P4DvAy4qizAFeVi+nIIQoAX4AHA8cB/wgyWD8UlGU+cDRwMlCiAvGuZ78qViY0SNwWEzYzIZ4IgzIPLQ+HVdlZkPQ8jYgYNbx0hD0NmWU69VQuuRc3khJQz6vAofZIUv3gokcgZakG0toCKRX9ONLFh9w4av4/GcNLZ8yGRpDyai9BJVKB73+kBxRma1iKDAA0VBuj0Db7e9dA61rh4eFQMbdHWUZE8bF2RRIY1HZoVyZaKzrU+UlxrSDHMkQ9O+RJ9OS+oxrVBRpiDSPwBcLplTFecNeXEa7rJhyVdI9Do/AoJa7tndmCb962lLlIGaskHIfefYTtPZJ+en6gpg0fhkGKeWDW1WBjYeWy+bI93nHC5nvkJZ7zDQy9UAwXkNwCfCA+vcDwKUZjjkPeFFRlF5FUfqAF4HzFUXxK4ryKoCiKCFgPTD6oNxYqVwk3bUMJ2UpM5G0u8mlM6SRTXiu5S1pAGyF6hdYydrDABDq2EZAMWMpHUVoSCgpHoFP1XIfa2hoytD0hrSTyWTMIMhE0lyC9oEA+wfyGFGZrV8EEoZg9R8AJbMh0I7L0EtQZNcm5aUZgt4mWeSQ5BGMRXAujq0gd/motovNYAiKkgboaANsPAaD9IJUvCEvboNcm+KsGFFwLvdapSHo6clSkDGYNkTG4pBJ4zwTxlrpaINBTTqP0RAYDIJiR1J3sckKiz8BW5/JbHTTytL9amjoQDNeQ1CpKIommNIOZNomTQeS5T5b1eviCCGKgIuRXkVGhBDXCSHWCiHWdnWNTUckhYqFgCJ3WGmUpQvPpdX6ZsRVKf+pySWpkRDsfU+qTUKi8iVHwjjSsZ0mZRqVhfnt5p1mJ0MoxJI8Ar9qCMYaGpoynOVyt619Ydo2yvd1FJrwY3veMmJGGzNEF1v2S82hnD0E2lqzoYV9tj8nj8sW2irLPNEsqwJpfAZBUmhoLDpDGiN5BDkNgXzOPn84vuHwpg2w94a9ONVTjNdcQjiqjNsQ+Ad7ZY1+MtGwNNDpQ2RqT4b976cUUmRjj2oIpkfUU9U4RqKmdBcDHHWVDC1veXL4wUnyEiA9goMyNCSEeEkIsTnDzyXJxymywHfkbqnhj28C/gbcqShKlkAaKIpyj6IoyxVFWV5enuNLmC/alynD7rzUlSY8F5eXyGUIKqQbnRzaaNsgPwA1J8nLRbNkaWGOElJj7052KdVU5PmFiQvPaRUbsVh8OtlYQ0NTRtLISkB6BJOdKAYQgljhLGaILta39AO5Skc1naEcoSGLU5UnUKDhnOxJx9IG6UWmnYzdVhMGkaFqqGOL7PJNOkn1jUV5VMNWKD+fGUpYAWkILO6MIdHiJI/AbrJjQKgyE0mGIOTFrZ4ReigCEuNgR79W2RPjwh+fy514ok5AGb5hqDlJVmbtXT3iw7f0+LGbjbg9TbIbWBs1OQZKnBb6kg3B9GPk/2zDX4cfnBYa8h2soSFFUc5WFGVxhp+ngA4hRDWA+juT37YPSJ4APUO9TuMeoFFRlN+M+VWMheJa2eafIU9Q4kz3CPIMDUFqeEjTRZ+lyiMIIQ1Qtsqh8BBW7152xqZTUTA6QxCv4Q555c6MQ9QjAPnFDgektzbZYSEVY0kNM0U36/fIyqGsoaG4IcgRGoJEeGhulrBQ8jFp4SGDQVDksAzPEXRskTHnJGny/nF5BEXyd7YS0r7d8oSYQXhNMwR9/jBCCJxGqzqcJnGS9oQ9ceXRDkU+11hzBJpHkLFySFNxLUjzCGYeLw1nHnkCrXRUdO+QwnX5zubOQInLklp+LgQc9WnY++7wUKC/d5ghOCg9ghF4GrhG/fsa4KkMx7wAnCuEKFaTxOeq1yGEuBUoBL4xznWMHoMRKuZnrRxKnsCEr1uWm+aSbIgPsU9KZrW8LUtVk3dUVUvkc8YyNAv17EKgsEuZRrk7+xyCZOIeQXRIxtaDHvwGgRGBzZjfYxw0JAvPdW6Ru7nJThSriKJZzDR0sVM9yWQNDfk65cklQ0llCmVz5c6y/ozsx5RqchQZKofs5uFTytIqhkDuyLWcwqjROs+zhYd6m7LujNPDV26TXQ6wV8dVKoqCL+zDFQmD2UFPSB5fMlaPQF1rgfCzqzPNI9AMQbpHYHXDtKPymk+wt9fPrFKHLB0dQyNZMsNCQwBLr5DnkGSvIBKUjaBJOQJfKIrzEMwR3AacI4RoBM5WLyOEWC6EuBdAUZRe4BbgPfXnx4qi9AohZgDfBxYC64UQG4QQ145zPaOjYlHmXgKnlVA0Fm/3xt8jv/i56oq1HaImSBaLytI1LSykUbVYus+Zug3V0tE9hhkU2PLbFcQVSEGqPwYH8Qk5nSzTOM6DmuTQUJuaRzlAHgHFNRTixaXICVVZu1+1WcUj1Zif+h/wmcczNmPFKamTJ4dMvQQOc2qOIDAgq3iSDEEoEsMXio6tmQxShOeGEY3IzucM+QEgPiNZ81pcJoc6wF4a0qHIEDElhjscAFcFPepxJeMJYwEz7GGautM8gkHNEGSQZKk5GfatS0lip6MoCnt6/dQVmaTxG4O0RDIlTgsDQ+HUzvCCatld/sHfEnnEDJGGQ9IjUBSlR1GUsxRFmaOGkHrV69cqinJt0nH3KYrSoP78Sb2uVVEUoSjKAkVRjlJ/skggThKVC+VJJ22IRbyXQAsPpSV0MpIeGmrfJK19zcmpx8UTxhn6Cbp3EEPgc9fkfRJPlqIm6IXAIF6DwHmoeQOg7rKFagg+kF/+ovyqp8aNWjk0XXQzrcie/f33dmbvKk6mYBrUn577GJNVPm82jyA5NNQpFWmTS0e14TXjShZDZo9gsFX2v2QxBEIIiuzmeEOby+xMkaLWJKidIT+4KuMx8zHnMywuEAZmOcKZQ0MGU+bO/9pTZAFC63tZH7rLG2QoHGWRtUvm+caRKIZEd3FferL/qE/LITXaTOU0eYlYTMEfih56huCQp0KVmkhTIi1V45jxhHEunSENq1vmHLQYsuaOavkBjfIFMrSQxRB0GSspKshfPjp5OA0hLwQH8RsMuA61RDFILR5HqXwPtUTxgfJqkuSos84hgHhz1IRRmrlyqNhhSZ1SlqViCMZxcs2lQJqjYkgjeYBO+rjK+FCagAdcFfT6QritJiymMZ5yDAawuqmyBtnV6SMWS6pL8bRLaYlMXtrM4wGRs4xUKx2dHS8dHV9oqERtKhsmUzPvQml8tfBQmkfgV6uhXAdYeRSOdEOQpXJIs+jdyR7BSFo3QqR2F7e8JRPShWnDLcw2mfDLVDnUvYNmpuddMQQJYblkQzDa6WQHFa4KucPr2HLgwkIQ9zxmiK7sk8lADQ2NkCgeDVovQZqqZmH6lLKOLfIkkjQspW+syqMauTyCvAxBwli5LAUp5aPamEqXvx9cVfT6QmPPDyStt9wcYEjtAo7j2Z/aVZyMvUh64TkSxlrp6LTwHkAkcjdjpNgp/x/D5pqYbak9Bdl0hg50IydHuiFwVUi3bJhHkCY8l49HAGp3cYf8Ure8PTwspJFJaiIWg+6dbI9UUVmQf1hHa+/3iURoaCzTyQ4anGVSPjgaPLCGwFFKxGhXPYIc8hK+rpErhkZDaUPG+cVFdgueQISIFmfWZhAkeUhaDmHUYyo1chqC3dLDdWXv4Sh2mOnzqR6BtUDdjEhDEB9KExiUoaGxjNPMsN4ig4z1p4SHtLGS2ag9RYaGIpmVS1t6/AgBhd4mKJopm9HGQVxmIt0jgNSeAl+qIZgqCWo40g0ByDxBmkegJQp7vEF5gs6lPJqMJjzXtR2GerMbgsrFMlaYPDhjYC9EhvgwUj2qpptMoSGfQeC0jH4WwUGBsyJxYjoQPQQaQhByzWCG6GZaVnmJflVeYoINAQzLE6QokCpqN/qwiiF5Eh71mEoNNe6e1SMoqcuZFJclrqpHYCtWO4vl7jo+iyCmxENDY5GfTsFaiEuRj5+iQpouL5FOzcmyI3vfuow37+n1U11gw9jTOO5EMSTOH8NyBJDaU6B5BKrM91RJUINuCGTlUNe2lHJOq8mI22aSoaFAvyxjzMsjUENDmhuaXjGkoQ1hTw4PqXHiXbFpowoNWQwWTMKoJos90iMwGEY9pvKgQTvJmuyJaV8HCEtZLQvtfZwyJ4vR18J+Exkaik80y2wI+ofCsloo5BluCNSwTPFYT7BCyLLMTH0EvbsTgmlZSC5xdVkKCAtBMCiNiuYRuGMxcFVOjCGwFWKODOK2mhIeQcgvDVkuQ1B7skwm73g+4817e/3MKrbKqr2y8eUHIKEVNWymCaT2FLSukSXp6pwKX1DmCJx6jmAKqFwodzFp5ZzxWuB8uoo1NJmJ3a/JdvcMqo1AYqebHB5SS0d3KdOoGEVoSAiBw2THpzXzBAfxGYyHXjOZhladVbU48wSwScRUUsssQxczirOEBvJtJhsN2vzi7nRDoHXuhhO9LhWphqDPH8ZkEOOrO88kMxGLJZrJclDslGMZg5FoXG/IqxoVzSNwxmIoE+UR2AoQAQ/1Fa6EIfDkKB3VsBdD3enw4VMZR8Xu6fVzlHtQhiPHqDGUjMlooMhhHp4s1tB6Cna9klKNGPcI9BzBFKB9udL6CUpdVqkpHlcezcMQOMsBRWrQ15yUveLFVSF3lckdxt3bCVkK6cU9Ko8ApLicbObxogQG8QlxCBsC9SR7IPMDGkWz1CEi/Zlv902CITAYUgfZaEuxa6GhUJIhWJByjOwqtoyvXySTIfC0yVBKjkQxJHktyXpDqicQNwSKQsBaSjASm5AcAYEBZpc72dHhlQ2f2ZrJ0ll4iRwClJabC4SjdAwGWWxRB9uMs3RUo8RpyW4ItJ4CSO0hCOmhoamjYj4gMlYO9XhD+QnOaWgniLA/e1hIo2oJdCR7BI30OeoAMWpD4LCoMwmCXgKBAWLiENQZ0tD6MQ5kfkBDLSFlYG/m2/PRGRoLpQ3D5Kjjom6+sAwhFteBNVVNtt8fGnvFkIatcHj5aB4VQ5AsMxFKGAK1Wsgb8uIUZgxADzJMWToBoSGCgyyfVUSXJ8jOTm9iMlm6vEQ68z8id+EfpoofxAfW0yqvGGfpqIaMKGQZqwkyPATDJKhBTxZPDRanDOFk6CXo9oby0xnSSD5BZEsUa1Qths5tCcGv7h20m2dhUmVsR4PT7MSrzi0e73SyKWfGcrlbyibdPJkUq81rncMVaQFpCAym3FIjYyHD/GJNNmLAH5AKmmn5AYCdnV5qSsZp8DN5BHkagqKkSWrx0JDaP+AJeXAJA9hL6FUrPcecy9CwFgAKK+tkVdeq7V35ewTOMlk9lBYe0kpHq0J75Pc3w3zmsZDTIwDZU2AvSSkH9qs5AoeeI5giKhfBvvdTJKRLnRZ6fUFivlEYAm036ygdOdZYuUR2bnbvkHkIXxctYjrlbuuoh4w4zU786txiv5qsO3RDQ2Xw2cez14VPJlVL5Rfzg79lvj1feYnRos0v7m+JX+W2SQXSuY33yusXfSzlLr5ghJ1dXpbMGGdRQDZDYDBDYe7xICkzCeIegSzv9IV9UnlUTRQDlDgnwHsBplmDzKlwsWpHp5SXMDsTukm5WPBR6XklSc+39EhDUOBtmpBEsUaJ05rbEJhtcN2rcOZN8au8eo5gill8GQzsgU2Pxq8qdVmIKRAc7JLJPEseJ1bNI5h14sgdsclSE2rF0I5RVgxpyLnFMlms7cgO2dDQVGIwwtGflUm8vpbht/s6JzY/oKGVkCZ1GBsMghNtezhxzz2w6OPyM5rEh22DKAosnSxDUFw7YrJea5zqS/YI1LnFnrAHZzQGrop4GeWE5AgAAgOsnFfOe7v7iAzsl95APnmSBRcDAj58On7Vnl4/LqsRU2/jhCSKNUqcZvr84dQO6HSKa1O0qHzBCHazEeMBnlcMcOBNz8HIwkuh+jfw6q2w6FIwWeMyE+HBTuxp3oA3GOGOF3fQ5QkSjSlEYjH1t8J1jjMpnX4ZI36kShvAaJXx35jcCWwOVFJRMXqNIDmlTCaLZWu//dANDU01R38GXvs5vP9/Kbs1YOK7ijUyzS8O+biNOxkwlVJy0a+Hneg2tsqT9+Lp4zQE1gJZmhqLJk78vSNXDEHyJLUQbrOsfvHGZFzcF/JRGA1DQWW8jFJrtBozSZIYK+fN449v7MbTtYfiXBVDybirYNYJMjy08kZA5giWFQUQA4MTligG6RFEYwoDQ+G8Q2K+KdIZAt0jkBgMcNYPZL32uvsBKFP/eVFv9zB5iV//awf3vbWbTfsG2N7hoaXHT9tAgD5fiC94vsRv99aO/JxGkyxdbd8kw0NGC5t8BWP3CNS5xYfsdLKDhaKZ0HA2vP8XqcCZjHeCdYY04vOLkwzB899lutLG74u/nTFuvam1n6oCGxV5ypVnRdtla70EiqKWjubODwDYLUasJgMD/vCwAfbesEcqj7plV7HRIHDnqag74loDAyyvLcZhMaIMto0YRozGFB54u5m7V+3kHesp0LmFf7zyOo+ta2Vbu4flDrUIYIISxZBIjPdmairLglQePfD5AdA9ggSzz4TaU+G12+GoT8c9Avy9UJKo9d3aNsgD7zRz5XGz+OnHlgx7mG889D5v7uxBUZSRy/oqF8O2f4DZTqxkNj17Y2P6YmtzixVfN15kY5weGhoHx34OHr4KGv8F8y+U18ViqrzEBEzHy0RpQ8IQbH0W1j/APwuuYLWyIOPhm/YNjD8/AKkyE/Zi+RpD3rwMAchwT58/hNlgxoYBnyI9Ak/IgzMih9b3tocpdphHnfsaRtL8BKvJyEn1JTibu1FcVeR65Bc/bOcHT8tikGpm8Y4Ntrz8F+6OyiGLy6ZphmD8XcUaWs9Ery/E7Dw/Mr5gZEryA6B7BAmEgLN/KMtF3/19/B9pCiR0hhRF4eanNlNgM/Ff52Z2I0+oL6XbG2RXcgt8NqqWSCmKPe8QKJJx4nwnkyXjNDuJAYHB/TJXQEKMTmcMzD1P7vzXP5C4LtAvk/uTERoCOb+4Z6csh3z636F6Ga9O++LwKWWAJxCmqdvHkvGGhSDJEKgeQZ4VQxpFDnNc/M5lsOBRIqAo+ELepK7i4PibyWDYRLVz6m1YCdFjzF3I8eDqPVQX2tjyo/N44YdXEq4+lm9M38pr/7WSl/7jdFaW9MoRshPo7SVkavL3CLxTNKYSdEOQyozlMP8ieOtOihlECLCE+uKG4PH1+3ivuY/vXDA/a9zvhHp57LtNPSM/n5YwDgww4JQx2TGFhjQF0rAHv+qFaMk7nTFgNEtxsMZ/wYA6VXUyuoqTKW2QgoWPfl4OUfn4vbgcjox6NVv2y0TxxHgEaVPKxmAINDlsl9GCT0A47CcQC+FS1GSxLzz+RHGGtZ5eLUN3Gweyq8W29Ph4o7GbT62YhdNqosBmxrz4EiydG6kxdNFQ4cLQvUOGhSZQ8nyYcGUeyFkEUxMa0g1BOmf+N4R9mN7+DeV2gTXqA0cpA0NhfvbPrRw1s4jLj52Z9e41pQ6qCmz5GYKk2vAuq2xmGktoSAsD+YQh7hHYTTmklHVG5pjPyiElGx6UlyejqzgZrXKo5S0471Yon0uRw5yqQKqyeZ88EU6sR5BkCIQRCrN/xpPRQkMALqMNj8GA1yfHtbo0j8A/AfISIA202RFfa5WQ8i9vdWQvS/3rmj0YDYIrViS9ngUflb+16qGu7ROaKIbk0FCOprI0fMEIDt0jOEiomA/LPg1r/sjxNrXb0FHKHS/uoMcX4tZLF+eMdQohOKG+hHebehMzj7NhK4x3s+4xyJrtsYSGtAohn0HgNQgcRisGof9rx0VJPdSvhPV/lhU1k9VVrKHVsM89H5Z/AUg0bA0GUpPWG1sHmFZoo2ysg+CTyWQIimaCKb8Td5HDEg8NOY02vAYDXrUbXyqPyulkE2IItPVqa1W7il/db2IoFB12aDAS5dG1rZy9oIKqZEXZkjopYbL1aRjqk0Z+AhPFIIUrXVZTZinqLHiDEVx6juAgYuV3AIWvh+XkzL1BO39+p5nPHF+TV7ne6PIEUkphV6waIcbWhp8sRe03GHCZ9ETxhHDMNVJuYterScqjk5QsLpsLl/0vfOx/4iEKLfzYnxYe2jxRiWJIJGCDSTmCEVRHkylSZysrioLb7JSbkSHpDbsUQcxaSN9EeQTaeuOGQHYV74sUZPTAX9jSQY8vxKePzzDudMFH5YyCXa/KyxPsEUAe3cVpTNWYStANQWaKZsKKLzInvB2AP64bpMhh4T+zJIjTGVWe4KirYMUX2e8zUOayYjKO/l8SNwTCgE8cwtPJDjbmf0Tmh9bfL+P3BvOESRAMQwhY8omUx9fmDGhSzwCDE5kohpRKHECdQ5BffgCk5HIkpuALRXGZnTI0pBkCayEDgSgxZQKayTRUvSEABttQ7CUIs41V2zuHHfrX1S3MLLFzakMGWfGFl8rfb/xa/p7AZjKN4lEYAkVR8IWmrnxUNwTZOPU/CBjkzvqddsF3LphPYZ4CX6PKE8y/ED7ySzo9gTEliiEpR2AQeA0GnHoz2cRgskpxsO3/lI1/rooDN0OZVAkHjXh+YEbRxDyJ0QQWtzo6sVf+HoUh0NbY5wvF5xZ7VZkTl70kXkc/WaEh4a7mxPpSVu3oSjlsZ6eXd5t6+fRxNZlDuWUNcmZ5xyYw2RKCgxNI6SgMgT8URVGmRnkUdEOQHWcZ62deQ1AxUTW9hk8ck1t3JZlR5QlUOj3BMRuCRI7AgN8gDt0xlQcjx1wjO793vjR5YaEsJIu6aUxoolhDUyDtVWdyjMojSMxNcFnkuMrBQB8ALkdZks7QRBmC5NCQlJdYOa+Clh4/zd2JUOxfV+/BbBRcvjzH93ah7COgdM6kzL4YTWhoKqeTgW4IctJ11Nc4O3In3/n4iaNuhhlVngDNEIytS1QLDfmFwGcw4dRLRyeOsjkJJdnJShRnIfkkq7GxdYDpRfaJO7GCenLtH3XpKCTJZftDuNSpeJ1DcnfuclZOgiFIks32tENBNSvnSQOthYcC4SiPrW/lvEVVuRPqWvXQBCeKNbThVvlsBn1qstulh4YOPi5eNp2nv3c5i6aNfvc1mjxBNKbQ4w2OqWIIZKmoQIaFfMZDeDrZwcox18jfk9VVnAW3zYQQqTmCzfsGxi80l44WbultAkT2yXoZKE4aqelSG77ah2TVkNs9nT7fOMdpZltrLCrzNu5qakqd1JY64uGhf2xsY2AozFWZksTJVCyAY/9NTgybBEqcFkKRWPwknwvNI3DoVUMHHwaDGPMHuKbUQXWhjXfyMAQ93iAxZWzNZKCNq7ThM4hDezrZwcrCj8q6+srhkiKTicEgKLSb4zmCAX+Y5h7/+IXm0kk2BAXTpURynqRIUdukJlf7UBdmRcHiro6XT5ZMVLLYWiA7vPtbZJ+HOodg5bwK3m3qIRCO8uDqFurLnZxQX5L7sYSAi38jO8kngXgvQR7dxVM5lAZ0QzBpyDxBKaubekZ0DTs9sumkfBwCYk6TE7/BgO9Qnk52sGK2ww0fwAlfPuBPXWQ3x0NDm/fL2PikeATBQbViKP/SUUhUNvX5wnFD0Bbsj8tL9PlC2M1G7OOZq5y+VpBNYCBnPgOnzysnEI7x53eaWb+nn08fN2t8IzwnAK27OOekMhU9R3AYc0J9Cd3eUGLQdhY6BuUIp7GGhkDmCfoMBsLiEJ5OdjAzCcnEfCh0WOKhoU2TkSiGRG3+GAyB2WjAbTXRPxTCpZa+tscCE99VrBE3BOpwGdUjOLG+FKvJwC9f2IHFZOATx+Zf3DFZlKiy2/kkjPUcwWGMlid4p6k353GaR1BZMA6PwOKiyyQ/RHpo6PCh2GFmQA0NbWodYGaJPR6OmTBshTDULwUXR5Eo1ihySq/FbZE9CR6h4Iwpqs7QZBkC1SNQZxXbzEaOry8lFI1x0ZLqiX+PxoDWHJpPd7GeIziMmVUi8wQjJYw7B9XQ0DgkA5wWF51G+SFy6J3Fhw1FdnPcI9i4r5+l04sm/klshYAavhyDIdD0hpI3IDI0VEGvLzRxiWJI9QiEIaWk96z5UgfqqhMmvidgLCRLUY/EVIeG9HkEk4iWJ3ijsSvnfIJOT4BihxmLaex22Wly0mMyAYquPHoYUeSw0OcL0e8Psbd3iE8fN0IlzFjQVD1hTIag0C6lqN1J/SsuDGBx0usPUVc2gR6q1gndtUOW8yaF7K48bhaLphVwbM0ISeIDhMNixGIyxCuncuFTB9c7JyqXMkp0j2CSySdPMJ4eAg2n2UlU3dXpswgOHwrtZgYDETbs7QcmIVEMiV02jEpnKH4Xh4UBfwi7yY5BdSxcRund9vnyH9WYF9pawz5IG1FpMRlYXntwGAGQG0Gtl2AkfKEIVpNhTBIzE4FuCCaZfPIEnZ6x9xBoJFcK6VVDhw9anf7bu2R4cfEYelpGRDu5uirBOnpvslgdTiOEwKnOCnOZnAQjUbzByJiEFEdcKwwzBAcj+XYXT+VQGtANwaSTT56gazBA+Rh7CDSSK4X0ZPHhg5b0fH1HFzWljrz1rkaFdnIdgzcAco2DgTDRmIJbqAULFhd9PpnbmFCPwGwHg3rCHGFW8cFASb4eQTAyZfkB0A3BpDNSP0EsptDlnZjQkIZePnr4oJ34t7V7Jr5sVEOVhhhLfgCkzISiwMBQGKeQJzO3tTAhLzGRFTxCJAyXWjp6MCOF5/LpI5g6CWrQDcEBIVeeoM8fIhxVxtxVrKGHhg5PNOE5mIT+AQ17kfw9RkNQnNRd7DbIv132konXGdLQEsaHRGjImldnsRxcPzWJYtANwQHhxHqph54pTzARPQSQ6hHooaHDh2Qd/wkbRpOOswwuuRuO/dyY7p4Qngvj1AyBo2ziJag14h7BwW8ISl0WfKEogXBuvSE5i0D3CA5rZpbYmVZo409v7eb5zW0pM2g1QzDeZLEWDrIZbZgMelXw4UJRUk5gwjWGkjn6qjGL6qXoDZnkhsblrJp4wTmNQ8gQ5NtL4DuUk8VCiBIhxItCiEb1d8bxTUKIa9RjGoUQ12S4/WkhxObxrOVgRgjBzRcvIhiO8eW/rOf0X6zif17bRb8/RKcmLzFBoSE9LHR44baZEQLqypwU2CYhUTwBxBVI/WHcRjsAblVwTojU8NaEoPU9HCLJYsjHEESnbDoZjL+h7DvAy4qi3CaE+I56+cbkA4QQJcAPgOXI9sV1QoinFUXpU2//OJBbjOcw4PzFVZy9oIKXtnbywNvN3PbPbfzmpR3MLJYn7olKFuthocMLo0FQ7LBMXn5gAohPKUvqLna6Z9LnC1FoN098bbytUE4VU2WvD2ZG4xFMlbwEjN8QXAKsVP9+AFhFmiEAzgNeVBSlF0AI8SJwPvA3IYQL+A/gOuCRca7loMdkNHD+4irOX1zFtvZBHni7mSfe30dlgXXc6oxaE5luCA4/fnfl0cwsOXg9vQKbCaNB0O8PUzjtWOjbgLu4ll7//omtGNI4+mqoWnpAx4aOlXwMgTaveCpDQ+N95kpFUdrUv9uBTCOcpgN7ky63qtcB3AL8CvCP9ERCiOuQBoNZsw4OLZHxML+qgJ99fCk3nj+foRESSfmgyUrohuDw46RMw9cPIoQQqsxEiJOXHM/8trcosZXQ52ue+EQxwKzj5c8hgNYf1OXJXkIaCMeITeG8YsjDEAghXgIyFex+P/mCoiiKECK/Ab3ycY8CZiuK8k0hRO1IxyuKcg9wD8Dy5cvzfp6DnSKHhaIJeBwtN6AbAp2poMghxfFOnHYif5/2d0Dugg9mT+ZA4LaacFiMtA0Esh6TGEpzEOcIFEU5O9ttQogOIUS1oihtQohqoDPDYftIhI8AZiBDSCcCy4UQzeo6KoQQqxRFWYnOqNEUR3WdIZ2poNhhiU9S0+j1hVg2o2hqFnSQIISgqtAWnzmSiamWoIbxl48+DWhVQNcAT2U45gXgXCFEsVpVdC7wgqIov1cUZZqiKLXAKcAO3QiMHZPBhN1kx2nRDYHOgafIbo5LSoCMe/f5J1iC+hClqsBG28BQ1tu9UyxBDeM3BLcB5wghGoGz1csIIZYLIe4FUJPEtwDvqT8/1hLHOhPLVQuu4uxZWR04HZ1Jo8hhYWAoYQg8wQjhqDKxgnOHKNIjyJ4j8Menkx3EOYJcKIrSA5yV4fq1wLVJl+8D7svxOM3A4vGsRQduOOaGqV6CzhGKVCBNhIYmrZnsEKSqQIaGYjEFg2F4pVM8NDSFOQK9s1hHR2fcFDst+ENRghG5u03oDB2cTXAHkupCG5GYQncW8blEsvjQDQ3p6OjoUGhPdBdDsiEYX8f84YCmI9aepXLIHzr0cwQ6Ojo6SQqkaYbgIBgiP9VUF0rZjWyGwKuOqXQdwlVDOjo6OnG9IS1PoP0u1kNDVBZKr6g9SwmpniPQ0dE5LCiMC89JA9DrC2MxGqY07n2wUOa0YjKIrB6BLxjBYjJgnqJ5xaAbAh0dnQmgOC48p4WGghQ7zYhDQA9osjEYBJUFtuyGYIp1hkA3BDo6OhPA8BxBOGWozpFOVaEtR2hoaiWoQTcEOjo6E4DdYsRqMsRDQ33+0OQIzh2iVOXwCLzBCM4pTBSDbgh0dHQmiKKkprJen24IktE8AkUZrpfpn+IxlTB+GeqDhnA4TGtrK4FAdnEnnSMPm83GjBkzMJv16pXJRgrPJcpHdUOQoKrAhj8UZTAQifdcaHiD0WHXHWgOG0PQ2tqK2+2mtrZWT1DpAFL4rKenh9bWVurq6qZ6OYc9RQ4z/f4wkWiMgSE9R5BMZaFsKusYDAw76fuCEaYVjm9C4Xg5bEJDgUCA0tJS3QjoxBFCUFpaqnuJB4giu4U+fyheOVTq0g2BRrV6os80l8AXnPrQ0GFjCADdCOgMQ/9MHDiKnWb6/OFEM5nuEcSpUmUmOrIYAr18VEdH57BASlGH6PFqOkO6IdCoKJDdxekegZxXrJePHlYYjUaOOuqo+E9zczOrVq2isLAwft3ZZ5/N9ddfz1FHHcXChQux2+3x2x599NGUx3vyySf58MMPJ2x9v/vd72hoaEAIQXd3d/x6RVH4+te/TkNDA0uXLmX9+vUAbN++nWOPPZalS5fyzjvvABCJRDj77LPx+0ccM61zhFHsMBOOKrT2yc+GbggSWE1GSp2WYb0EwUiMaEyZ0ulkcBgliw8G7HY7GzZsSLmuubmZU089lWeffXbY8c3NzVx00UXD7qPx5JNPctFFF7Fw4cIJWd/JJ5/MRRddxMqVK1Ou/+c//0ljYyONjY2sXr2ar3zlK6xevZo//OEP/L//9/+ora3lhhtu4LHHHuP3v/89n/nMZ3A4juxZtDrDKbLLE39Ttw/QDUE6mUZWHgwS1HCYGoIfPbOFD/cPTuhjLpxWwA8uXjShj5mLt99+m6effprXXnuNW2+9lcceewyPx8OXv/xl/H4/s2fP5r777qO4uJiVK1eybNkyXnvtNSKRCPfddx/HHXfcsMc8+uijMz7XU089xdVXX40QghNOOIH+/n7a2towm834/X78fj9ms5n+/n6eeeYZnn/++cl++TqHIEWq3tCuTm/KZR1JVYGN/WmhIb+qPKoniw8jhoaG4mGej33sY/Hr33jjjfj1P/nJT/J6rJNOOomPfvSj/OIXv2DDhg3Mnj2bq6++mp///Ods3LiRJUuW8KMf/Sh+vN/vZ8OGDdx99918/vOfH9W69+3bx8yZM+OXZ8yYwb59+7j++uv56U9/yjXXXMP3vvc9brnlFr73ve9hMOgfG53haNPIdnf7cFlNWE1TG/c+2MjtEUzte3VYegQHcueeTKbQEJA1NDQaBgYG6O/v5/TTTwfgmmuu4fLLL4/ffuWVVwJw2mmnMTg4SH9/P0VFReN6zlmzZrFq1SoAdu7cSWtrKwsWLOCzn/0soVCIW265hblz547rOXQOHzQp6pYeP1VTXBd/MFJVYKPXFyIQjmIzyxO/Tx1KM9U5An1rd5iQXiYphOC8887jqKOO4tprr81yL8n06dPZu3dv/HJrayvTp09POeb73/8+t956K3feeSfXXnstt99+e4pHoqNTqOYIQtGYPqs4A5px7EwaZK/NItBDQzpZcbvdeDweAAoLCykuLuaNN94A4P/+7//i3gHAww8/DMCbb75JYWEhhYWFvPDCC2zYsIF777035/N89KMf5c9//jOKovDuu+9SWFhIdXV1/PbXXnuNadOmMWfOHPx+PwaDAYPBoFcO6aSQnBMo0fMDw6iKN5UNxa/zadPJ9GSxTjY+9alP8cUvfpE777yTRx99lAceeCCeLK6vr+dPf/pT/FibzcbRRx9NOBzmvvvuy/h4d955J7fffjvt7e0sXbqUCy+8kHvvvZcLL7yQ5557joaGBhwOR8rjKorCrbfeGjc01113HVdddRWRSITf//73k/sG6BxSmI0G3FYTnmBEn1WcAa27OLmEND6dzKLnCA4bvF7vsOtWrlw5rFxTo7a2ls2bN2d9vJNPPnlYH8G7776b8djPfOYz/OY3v8m5vq9//et8/etfH3a9EIK77ror432EELz44ovxywsWLIj3GejopFPoMKuGQPcI0sk0xP5gKR/VQ0M6OjoThiYroecIhuO2mXFZTSkegT90cOQIdI/gMECr7NHRmWq0PEGpbggyUllgTfMIoliMBiymqd2T6x6Bjo7OhBH3CHTBuYykj6z0BSM4priHAHRDoKOjM4FoHoEuL5GZqgJ7ikfgC039mErQDYGOjs4EUqTnCHJSVWil0xMkGpMjKw8GCWrQDYGOjs4EUlPiwG42UuHWy0czUVVoJxpT6PbKpjJfcOolqEE3BBNKa2srl1xyCXPmzGH27NnccMMNhEKhjMfu37+fT3ziEyM+5oUXXkh/f/+Y1vPDH/6QX/7ylxmvnz59OkcddRSLFy/m6aefHtPjTzQ//elPp3oJOuPk0qOn89q3V+K26eWjmahKKyH1HgTTyUA3BBOGoih8/OMf59JLL6WxsZEdO3bg9Xr5/ve/P+zYSCTCtGnThs0fyMRzzz03bs2gTHzzm99kw4YN/P3vf+fzn/88sVgsr/tFo9EJX4vGWAzBZK5HZ/QYDYIKt64zlI30kZX+gyRHMPUrmAz++R1o3zSxj1m1BC64LevNr7zyCjabjX/7t38D5JCaO+64g7q6On70ox/xyCOP8Pjjj+P1eolGozzwwANcdNFFbN68Gb/fz+c+9zk2b97MvHnz2L9/P3fddRfLly+ntraWtWvX4vV6ueCCCzjllFN4++23mT59Ok899RR2u50//vGP3HPPPYRCIRoaGvi///u/vOcFLFiwAJPJRHd3N9dddx179+4lEAhwww03cN111wHgcrn40pe+xEsvvcRdd93FK6+8wjPPPMPQ0BAnnXQSf/jDHxBCsHLlSo4++mjeeOMNfD4ff/7zn/nZz37Gpk2buOKKK7j11lsB+Mtf/sKdd95JKBTi+OOP5+677+b73/9+XL110aJFPPjggxmPMxqNw9bz7LPP8vTTT2MymTj33HMzekE6OgcDWlOZpkIqQ0NTfxrWPYIJYsuWLRx77LEp1xUUFDBr1ix27twJwPr163n00Ud57bXXUo67++67KS4u5sMPP+SWW25h3bp1GZ+jsbGR66+/ni1btlBUVMRjjz0GwMc//nHee+89PvjgAxYsWMD//u//5r3u1atXYzAYKC8v57777mPdunWsXbuWO++8k56eHgB8Ph/HH388H3zwAaeccgpf+9rXeO+999i8eTNDQ0MpyqoWi4W1a9fy5S9/mUsuuYS77rqLzZs3c//999PT08PWrVt5+OGHeeutt9iwYQNGo5EHH3yQ2267La7e+uCDD2Y9Ln09CxYs4IknnmDLli1s3LiRm266Ke/XrqNzoCl1WjAbRdwj8AYjUy5BDYerR5Bj5z6VnHPOOZSUlAy7/s033+SGG24AYPHixSxdujTj/evq6jjqqKMAOPbYY2lubgZg8+bN3HTTTfT39+P1ejnvvPNGXMsdd9zBX/7yF9xuNw8//DBCCO68806eeOIJAPbu3UtjYyOlpaUYjUYuu+yy+H1fffVVbr/9dvx+P729vSxatIiLL74YkAJ2AEuWLGHRokVx8br6+nr27t3Lm2++ybp161ixYgUgZzhUVFQMW9/LL7+c9bjk9RQWFmKz2fjCF77ARRddxEUXXTTia9fRmSoMauhM8wj8oQiOg8AjmPoVHCYsXLhwWMx/cHCQPXv20NDQwPr163E6neN6Dqs1UYlhNBoZGpIqhp/73Od48sknWbZsGffff39encbf/OY3+c///M/45VWrVvHSSy/xzjvv4HA4WLlyJYGA/LDabDaMRrlrCQQCfPWrX2Xt2rXMnDmTH/7wh/HjktdoMBhS1mswGIhEIiiKwjXXXMPPfvaznOvLdVzyekwmE2vWrOHll1/m0Ucf5Xe/+x2vvPLKiK9fR2eqqC600TYwRDASJRxV9PLRw4mzzjoLv9/Pn//8Z0AmMb/1rW/xuc99bsR4/cknn8wjjzwCwIcffsimTaPLb3g8HqqrqwmHw/HwyWgZGBiguLgYh8PBtm3bsorbaSf9srIyvF5vXgnvZM466yweffRROjs7Aejt7aWlpQUAs9lMOBwe8bhkvF4vAwMDXHjhhdxxxx188MEHo1qPjs6BprLQRsdgMC5B7Zxi5VEYpyEQQpQIIV4UQjSqv4uzHHeNekyjEOKapOstQoh7hBA7hBDbhBCXZbr/oYAQgieeeIK///3vzJkzh7lz52Kz2fKqhPnqV79KV1cXCxcu5KabbmLRokUUFhbm/dy33HILxx9/PCeffDLz588f0/rPP/98IpEICxYs4Dvf+Q4nnHBCxuOKior44he/yOLFiznvvPPioZt8WbhwIbfeeivnnnsuS5cu5ZxzzqGtrQ2QEtdLly7lqquuynlcMh6Ph4suuoilS5dyyimn8Otf/3r0L15H5wBSXSA9grgE9UHgEQhFUcZ+ZyFuB3oVRblNCPEdoFhRlBvTjikB1gLLAQVYBxyrKEqfEOJHgFFRlJuEEAagRFGU7pGed/ny5cratWtTrtu6dSsLFiwY82uZSqLRKOFwGJvNxq5duzj77LPZvn07FovenTkRHMqfDZ3Dj3vfaOLWf2zl4etO4Ip73uXuq47hwiXVI99xAhBCrFMUZXn69eM1RZcAK9W/HwBWATemHXMe8KKiKL3qQl4Ezgf+BnwemA+gKEoMGNEIHI74/X7OOOMMwuEwiqJw991360ZAR+cwRZtUtqvLB0y9BDWM3xBUKoqi+evtQGWGY6YDe5MutwLThRBF6uVbhBArgV3A1xRF6Rjnmg453G436R6Ojo7O4YnWXbyrSw6yOhjKR0fMEQghXhJCbM7wc0nycYqMMY0mzmQCZgBvK4pyDPAOkLUTSAhxnRBirRBibVdX1yieRkdHR+fgQWsq29kpDYHjUOgsVhTl7Gy3CSE6hBDViqK0CSGqgc4Mh+0jET4CefJfBfQAfuBx9fq/A1/IsY57gHtA5ghGWreOjo7OwUjlMI9g6g3BeMtHnwa0KqBrgKcyHPMCcK4QolitKjoXeEH1IJ4hYSTOAj7McH8dHR2dwwaLyUCZy8K+ftkHdDDkCMZrCG4DzhFCNAJnq5cRQiwXQtwLoCaJbwHeU39+rCWOkYnlHwohNgKfBb41zvXo6OjoHPRUFdrQCjYPeRlqRVF6FEU5S1GUOYqinK2d4BVFWasoyrVJx92nKEqD+vOnpOtbFEU5TVGUperj7BnPeqYal8uVcvn+++/na1/72oQ/T39/P3ffffeEPV5PTw9nnHEGLpdr2HrXrVvHkiVLaGho4Otf/zpaufGNN97I0qVLufrqq+PH/uUvf+E3v/nNhK1LR+dwRUsYmwwCi3Hq+3qnfgU6IxKJRFIuT7QhsNls3HLLLRlVO7/yla/wxz/+kcbGRhobG3n++ecZGBhg/fr1bNy4EYvFwqZNmxgaGuJPf/oT119//YStS0fncEUrIXVaTQghpng1h6nW0M/X/Jxtvdsm9DHnl8znxuPSWyTy55lnnuHWW28lFApRWlrKgw8+SGVlJWvWrOGGG24gEAhgt9v505/+xLx587j//vtTZKuTFUu/853vsGvXLo466ijOOeccbr/9dr797W/zz3/+EyEEN910E1dccQWrVq3i5ptvxu12s3PnTs444wzuvvtuDIZU++90OjnllFPiKqkabW1tDA4OxruMr776ap588klOOeWUeM+D3+/HbDbzy1/+kn//93/HbNYHkujojITmERwMiWI4TA3BVKHp6Wv09vbG1ThPOeUU3n33XYQQ3Hvvvdx+++386le/Yv78+bzxxhuYTCZeeuklvve978XlpbVdd7pi6W233cbmzZvZsGEDAI899hgbNmzggw8+oLu7mxUrVnDaaacBsGbNGj788ENqamo4//zzefzxx/OajAawb98+ZsyYEb88Y8YM9u3bh9vt5sILL+Too4/mrLPOorCwkNWrV/Pf//3fY33rdHSOKKoK7QA4DgKdIThMDcF4du7jQdPT17j//vvjjWKtra1cccUVtLW1EQqFqKurA6TY2zXXXENjYyNCiLjoGmSXrU7nzTff5Morr8RoNFJZWcnpp5/Oe++9R0FBAccddxz19fUAXHnllbz55pt5G4JcfPvb3+bb3/42ANdeey0//vGPuffee/nXv/7F0qVL9bkA/7+9uw+OqrziOP79gUCsgFRAhEQFASkMkCAvWoE2MJURtWI60je10kpbnUJ9hYozTFHbqSLVdqqMdKwTsGoDKgg6I1iRgq0gKNGkWIuAbaNUKFWKIojk9I/7bNgEQpYsm032ns8Mk3uffe7mPOTunr3P3Xuuc0eROCJoDt8YAj9H0GSmTp3KlClTqKioYN68eTVVPGfOnMmYMWOorKxk2bJltUo6p1u2Gjhs/jFRHK+oqIiioqKjXtGcn59PVVVVzXpVVRX5+fm1+mzcuBEzo1+/fixatIiFCxeyZcsWNm/enHbszuWqxDmC5jI15ImgiezevbvmTXT+/PlHbC8tLU3puTp06MCePXtq1kePHk1ZWRkHDx5k586drF69mhEjRgDR1NC2bduorq6mrKyMUaNGUVJSQnl5OeXl5Qwbdlj9qRrdu3enY8eOrF27FjNjwYIFTJhQ64JyZs6cyZ133smBAwdq7h/cqlUr9u7dm9JYnIujQyeLm8fUkCeCJjJr1iwmTpzI0KFD6dKlS0379OnTmTFjBkOGDDns20H16dy5MyNHjmTgwIFMmzaNkpISBg8eTGFhIWPHjmX27NmcdtppAAwfPpwpU6bQv39/evXqRUlJyRGfs2fPntx0002UlpZSUFDApk3RtX1z585l8uTJ9OnTh969ezN+/PiabZYsWcKwYcPo0aMHnTp1oqioiEGDBrFv3z4KCwsb+1/lXM5r3+4EOrQ7oVncuB7SLEOdLblWhjpTVq1axZw5c2rdUziOfN9wzdFj6/5J327tGd6z4fOAx0umylA755xrhG+fe0a2Q6jhiSCHFRcXU1xcnO0wnHPNXE6dI2iJ01wus3yfcK5hOZMI8vLy2LVrl7/wXQ0zY9euXeTl5WU7FOeatZyZGiooKKCqqgq/aY1LlpeXV+vqaOfc4XImEbRp06bmal3nnHOpy5mpIeecc43jicA552LOE4FzzsVci7yyWNJO4B+N3LwL8J/jGE5L4eOOFx93vKQ67jPNrGvdxhaZCNIhacORLrHOdT7uePFxx0u64/apIeecizlPBM45F3NxTAS/zXYAWeLjjhcfd7ykNe7YnSNwzjlXWxyPCJxzziXxROCcczEXm0Qg6UJJb0l6W9Kt2Y4nkyQ9LGmHpMqktlMkPS9pc/j5+WzGmAmSTpf0oqRNkv4q6frQntNjl5Qn6RVJr4dx3x7ae0laF/b5Mkltsx1rJkhqLWmjpGfCes6PW9I7kioklUvaENoavZ/HIhFIag08AIwHBgDfkjQgu1FlVClwYZ22W4EXzKwv8EJYzzWfATeb2QDgPOBH4e+c62PfD4w1s0KgCLhQ0nnA3cB9ZtYH+AC4JnshZtT1wJtJ63EZ9xgzK0q6fqDR+3ksEgEwAnjbzLaa2afAH4AJWY4pY8xsNfDfOs0TgPlheT5wWVPG1BTMbLuZvRaW9xC9OeST42O3yEdhtU34Z8BY4InQnnPjBpBUAFwMPBTWRQzGXY9G7+dxSQT5wL+S1qtCW5x0M7PtYfnfQLdsBpNpknoCQ4B1xGDsYXqkHNgBPA9sAT40s89Cl1zd538FTAeqw3pn4jFuA1ZIelXSD0Jbo/fznLkfgUudmZmknP3esKT2wJPADWb2v+hDYiRXx25mB4EiSZ2AxcAXshtR5km6BNhhZq9KKs5yOE1tlJm9K+lU4HlJf0t+8Fj387gcEbwLnJ60XhDa4uR9Sd0Bws8dWY4nIyS1IUoCj5rZU6E5FmMHMLMPgReBLwKdJCU+7OXiPj8SuFTSO0TTvWOBX5P748bM3g0/dxAl/hGksZ/HJRGsB/qGbxO0Bb4JLM1yTE1tKXB1WL4aeDqLsWREmB/+HfCmmd2b9FBOj11S13AkgKQTgQuIzo+8CFweuuXcuM1shpkVmFlPotf0SjO7ghwft6STJHVILAPjgErS2M9jc2WxpIuI5hNbAw+b2c+zG1HmSHocKCYqTfs+8FNgCbAQOIOohPfXzazuCeUWTdIoYA1QwaE549uIzhPk7NglDSY6Odia6MPdQjO7Q9JZRJ+UTwE2Alea2f7sRZo5YWroFjO7JNfHHca3OKyeADxmZj+X1JlG7uexSQTOOeeOLC5TQ8455+rhicA552LOE4FzzsWcJwLnnIs5TwTOOddMSZoYCglWS0rpnsSSDoZidOWSUvqavCcC16wl7dSVkhZJ+lwziKlY0vlpPkcPSU803LPWNpMk3R+Wr5X0nXRicM1L2K9K6zRXAl8DVh/DU30SitEVmdmlqWzgicA1d4mdeiDwKXBtKhslXVmaCcXAMSWCuvGY2Xtmdnl9/RtiZg+a2YLGbu9aBjN708zeqtseakvdI2m9pDck/TCd3+OJwLUka4A+kr4a6s1vlPRHSd0AJM2S9IikPwOPSOopaY2k18K/80O/Ykl/kvS0pK2S7pJ0haKa/hWSeod+XSU9GV5s6yWNDMXsrgVuDEcqo4/U70jxJA8kxFYZlidJekrSc4pqyc9O6vddSX+X9ApRSQWSnvuWsNwn/D+8HsaZiH9a0htF4h4FJ0l6NvStlPSNjPylXKZdA+w2s+HAcOD7knqFx/IkbZC0VtJlqTyZF51zLUL4RD0eeA54CTgvFNaaTFR98ubQdQBRQa5PwjTSBWa2T1Jf4HEgMc9aCPQnKte9FXjIzEYoupnNVOAGoro195nZS5LOAJabWX9JDwIfmdmcENtjdfuF564VTwNDLCKqlrofeEvSb4jur3A7MBTYTVQ6YeMRtn0UuMvMFkvKA1pJGgf0JapBI2CppC8BXYH3zOziEPvJDcTlMkzSOqAd0B44RVEVWYCfmNnyejYbBwyWlDiqPJno770NODMUpDsLWCmpwsy2HC0GTwSuuTsx6YWxhqiWUD+gTFFhrbZEO3/C0qQ33TbA/ZKKgIPA2Un91idK9kraAqwI7RXAmLD8FWCADlUv7aiosmldR+u3NIUkANENRXaHeDYBZxKVCFllZjtDe1mdMaCo5ky+mS0GMLN9oX0c0ZtFInG0J3qjWAP8UtLdwDNmtiaF2FwGmdm5UFMmY5KZTUphMwFTj5QokgrSbZW0iugDhicC16J9YmZFyQ3h0/K9ZrY0vHhmJT38cdLyjUS1lgqJpkH3JT2WXHumOmm9mkOvi1ZERx7J25H0hk8K/T6u27keyfEcJP3XpoBfmNm8wx6QzgEuAn4m6QUzuyPN3+Wa3nLgOkkrzeyApLOJqqy2Bfaa2X5JXYimE2cf7YnAzxG4lulkDpUWvrqBftvNrBq4iqgo27FYQTRNBEA4sgDYA3RIoV+61gFfltRZUXntiXU7hDuxVSXmgiW1C1Niy4HvJY5MJOVLOlVSD6I3it8D9wDnHKdYXQZIKpFURVRW/FlJiSOAh4BNwGvhXNM8og8P/YENkl4nmkq8y8w2NfR7/IjAtUSzgEWSPgBWAr3q6TcXeFLR1yyfI/VP5wk/Bh6Q9AbRa2U10YniZcATkiYQJYD6+qXFzLZLmgW8DHwIlNfT9SpgnqQ7gAPARDNbIak/8HI4MvkIuBLoA9wjqTr0vS7dON3xYWargFV12hZzqNJocns1UWXd2+o89Bdg0LH+bq8+6pxzMedTQ845F3OeCJxzLuY8ETjnXMx5InDOuZjzROCcczHnicA552LOE4FzzsXc/wFyDQF3PGAMwgAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(np.arange(100000,100050,1), reverse_top10fft[100000:100050], label = \"FFT top-10%\")\n",
+ "plt.plot(np.arange(100000,100050,1), conc2[100000:100050], label = \"Original Parameters\")\n",
+ "plt.plot(np.arange(100000,100050,1), reverse_top10[100000:100050], label = \"Haar top-10%\")\n",
+ "plt.title('Parameter Values') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.legend()\n",
+ "plt.draw()\n",
+ "plt.savefig(\"ParametersWaveletHaar.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 175,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar tensor(12.0875)\n"
+ ]
+ }
+ ],
+ "source": [
+ "wavelet = 'haar'\n",
+ "coeff = pywt.wavedec(conc5000.numpy(), wavelet, level = None)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 176,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar tensor(9.4642)\n"
+ ]
+ }
+ ],
+ "source": [
+ "wavelet = 'haar'\n",
+ "coeff = pywt.wavedec(conc5000.numpy(), wavelet, level = None)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.2*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 177,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "6\n",
+ "haar tensor(12.1470)\n",
+ "(52814,)\n",
+ "(52814,)\n",
+ "(105628,)\n",
+ "(211256,)\n",
+ "(422512,)\n",
+ "(845023,)\n",
+ "haar tensor(16.9818)\n",
+ "(52814,)\n",
+ "(52814,)\n",
+ "(105628,)\n",
+ "(211256,)\n",
+ "(422512,)\n",
+ "(845023,)\n",
+ "haar tensor(2.3811e-06)\n"
+ ]
+ }
+ ],
+ "source": [
+ "wavelet = 'haar'\n",
+ "coeff = pywt.wavedec(conc5000.numpy(), wavelet, level = 5)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "print(len(coeff))\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))\n",
+ "\n",
+ "reduced = []\n",
+ "for i, o in enumerate(coeff):\n",
+ " print(o.shape) \n",
+ " if i > 3:\n",
+ " reduced.append(np.zeros_like(o))\n",
+ " continue\n",
+ " reduced.append(o)\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(reduced, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))\n",
+ "\n",
+ "reduced = []\n",
+ "for i, o in enumerate(coeff):\n",
+ " print(o.shape) \n",
+ " if i > 5:\n",
+ " reduced.append(np.zeros_like(o))\n",
+ " continue\n",
+ " reduced.append(o)\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(reduced, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))\n",
+ " \n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# with resnet"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 178,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using cache found in /home/jeffrey/.cache/torch/hub/pytorch_vision_v0.10.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "model = torch.hub.load('pytorch/vision:v0.10.0', 'resnet18', pretrained=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 179,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "resw = {}\n",
+ "for k,v in model.state_dict().items():\n",
+ " resw[k] = v.clone()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flatr = []\n",
+ "for v in resw.values():\n",
+ " flatr.append(v.flatten())\n",
+ "concr = torch.cat(flatr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 190,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "bior1.1 43.02796936035156 115077.1953125\n",
+ "bior1.3 44.042415618896484 117252.8203125\n",
+ "bior1.5 44.798824310302734 119106.71875\n",
+ "bior2.2 44.7282600402832 117956.5\n",
+ "bior2.4 44.04926300048828 116528.5078125\n",
+ "bior2.6 44.09769821166992 116693.171875\n",
+ "bior2.8 44.207454681396484 116984.125\n",
+ "bior3.1 65.93879699707031 171188.890625\n",
+ "bior3.3 52.62727355957031 136826.25\n",
+ "bior3.5 50.10884475708008 130465.40625\n",
+ "bior3.7 49.333717346191406 128473.21875\n",
+ "bior3.9 49.07342529296875 127796.1640625\n",
+ "bior4.4 42.742881774902344 114508.9921875\n",
+ "bior5.5 43.98085403442383 118580.4296875\n",
+ "bior6.8 42.57365417480469 113812.046875\n",
+ "coif1 42.74231719970703 114690.046875\n",
+ "coif2 42.56312561035156 114001.890625\n",
+ "coif3 42.54220199584961 113903.578125\n",
+ "coif4 42.558956146240234 113906.859375\n",
+ "coif5 42.56120681762695 113924.28125\n",
+ "coif6 42.592472076416016 113980.0390625\n",
+ "coif7 42.60406494140625 114030.09375\n",
+ "coif8 42.61618423461914 114052.2734375\n",
+ "coif9 42.6173095703125 114053.078125\n",
+ "coif10 42.63934326171875 114081.7890625\n",
+ "coif11 42.65342712402344 114141.3671875\n",
+ "coif12 42.65858840942383 114128.3984375\n",
+ "coif13 42.66160583496094 114154.7890625\n",
+ "coif14 42.68099594116211 114187.8828125\n",
+ "coif15 42.693885803222656 114239.90625\n",
+ "coif16 42.69415283203125 114222.0234375\n",
+ "coif17 42.69487762451172 114230.65625\n",
+ "db1 43.02796936035156 115077.1953125\n",
+ "db2 42.69436264038086 114436.171875\n",
+ "db3 42.692832946777344 114409.2578125\n",
+ "db4 42.69171905517578 114270.2265625\n",
+ "db5 42.7407341003418 114356.6796875\n",
+ "db6 42.832889556884766 114583.046875\n",
+ "db7 42.90106201171875 114756.140625\n",
+ "db8 42.927757263183594 114787.8671875\n",
+ "db9 42.980587005615234 114925.3125\n",
+ "db10 43.0425910949707 115035.796875\n",
+ "db11 43.09166717529297 115145.9609375\n",
+ "db12 43.11075210571289 115177.953125\n",
+ "db13 43.153038024902344 115282.9765625\n",
+ "db14 43.23004913330078 115438.109375\n",
+ "db15 43.254371643066406 115495.4375\n",
+ "db16 43.26611328125 115499.40625\n",
+ "db17 43.29021453857422 115553.8359375\n",
+ "db18 43.339332580566406 115670.8515625\n",
+ "db19 43.363834381103516 115699.546875\n",
+ "db20 43.3875732421875 115747.3046875\n",
+ "db21 43.406944274902344 115809.53125\n",
+ "db22 43.44538879394531 115883.546875\n",
+ "db23 43.48051071166992 115981.046875\n",
+ "db24 43.502601623535156 115997.3203125\n",
+ "db25 43.519954681396484 116035.3359375\n",
+ "db26 43.53356170654297 116078.15625\n",
+ "db27 43.55718994140625 116114.859375\n",
+ "db28 43.56884765625 116135.7421875\n",
+ "db29 43.60223388671875 116207.0390625\n",
+ "db30 43.6269645690918 116254.625\n",
+ "db31 43.62778091430664 116261.8984375\n",
+ "db32 43.65283966064453 116317.5546875\n",
+ "db33 43.683868408203125 116371.8046875\n",
+ "db34 43.71052551269531 116451.03125\n",
+ "db35 43.69470977783203 116395.71875\n",
+ "db36 43.722896575927734 116448.71875\n",
+ "db37 43.732479095458984 116494.3203125\n",
+ "db38 43.75498962402344 116516.2890625\n",
+ "dmey 42.70671844482422 114251.421875\n",
+ "haar 43.02796936035156 115077.1953125\n",
+ "rbio1.1 43.02796936035156 115077.1953125\n",
+ "rbio1.3 42.904876708984375 114998.609375\n",
+ "rbio1.5 43.487056732177734 116603.4375\n",
+ "rbio2.2 48.12295150756836 128830.65625\n",
+ "rbio2.4 45.82650375366211 123338.625\n",
+ "rbio2.6 45.57454299926758 122649.625\n",
+ "rbio2.8 45.610652923583984 122736.25\n",
+ "rbio3.1 323.1489562988281 493834.4375\n",
+ "rbio3.3 63.0237922668457 168654.40625\n",
+ "rbio3.5 55.14375686645508 147856.1875\n",
+ "rbio3.7 53.33525466918945 143081.296875\n",
+ "rbio3.9 52.70606994628906 141449.8125\n",
+ "rbio4.4 43.57114028930664 116553.6171875\n",
+ "rbio5.5 43.684425354003906 115561.5078125\n",
+ "rbio6.8 43.01945495605469 115265.328125\n",
+ "sym2 42.69436264038086 114436.171875\n",
+ "sym3 42.692832946777344 114409.2578125\n",
+ "sym4 42.546844482421875 113956.859375\n",
+ "sym5 42.57182693481445 113959.1640625\n",
+ "sym6 42.52703094482422 113885.0546875\n",
+ "sym7 42.550376892089844 113925.90625\n",
+ "sym8 42.56036376953125 113907.7109375\n",
+ "sym9 42.602020263671875 114008.0625\n",
+ "sym10 42.568580627441406 113940.015625\n",
+ "sym11 42.623958587646484 114091.0546875\n",
+ "sym12 42.602752685546875 114007.203125\n",
+ "sym13 42.62556076049805 114069.9375\n",
+ "sym14 42.608367919921875 114038.21875\n",
+ "sym15 42.66935348510742 114179.7578125\n",
+ "sym16 42.629066467285156 114064.859375\n",
+ "sym17 42.639869689941406 114103.421875\n",
+ "sym18 42.6433219909668 114109.6328125\n",
+ "sym19 42.72231674194336 114301.625\n",
+ "sym20 42.658966064453125 114131.046875\n",
+ "min: tensor(42.5270) sym6 91\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "#wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " for v in resw.values():\n",
+ " flat = v.flatten()\n",
+ " #print(flat.shape)\n",
+ " lens.append(len(flat))\n",
+ " to_cat.append(flat)\n",
+ " flat = torch.cat(to_cat, dim=0)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 181,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "11699132"
+ ]
+ },
+ "execution_count": 181,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(concr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 197,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " concr.abs(), round(0.1*len(concr)), dim=0, sorted=False\n",
+ " )\n",
+ "top10_og = torch.zeros(len(concr))\n",
+ "top10_og[topk_og.indices] = concr[topk_og.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 198,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(47.3773)"
+ ]
+ },
+ "execution_count": 198,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(top10_og - concr, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 199,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "to_cat = []\n",
+ "for v in resw.values():\n",
+ " flat = v.flatten()\n",
+ " to_cat.append(flat)\n",
+ "flat = torch.cat(to_cat, dim=0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 200,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(71.3879)\n"
+ ]
+ }
+ ],
+ "source": [
+ "flat_fft = fft.rfft(flat)\n",
+ "topk = torch.topk(\n",
+ " flat_fft.abs(), round(0.1*len(flat_fft)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ "top10[topk.indices] = flat_fft[topk.indices]\n",
+ "reverse_top10fft = fft.irfft(top10)\n",
+ "fft_layers.append(reverse_top10fft)\n",
+ "err = torch.norm(reverse_top10fft - flat, 2)\n",
+ "print(err)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 201,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(42.6944)\n"
+ ]
+ }
+ ],
+ "source": [
+ "coeff = pywt.wavedec(flat.numpy(), \"sym2\", level = None)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10wv = torch.from_numpy(pywt.waverec(og, wavelet = \"sym2\"))\n",
+ "err = torch.norm(reverse_top10wv - flat, 2)\n",
+ "print(err)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 202,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACDNklEQVR4nO2dd3gc1dWH37NNWmnVi2VL7r3buGMDNsWmmBoIECAmJBACBJIvJIGEFGogkFACJBB6LzbFEKoBg8Hdxr3JVZZsq7eVVmV37/fHzK5X0qpZkmVZ930ePbt7587MHWk1Z+655/yOKKXQaDQaTffF0tkD0Gg0Gk3nog2BRqPRdHO0IdBoNJpujjYEGo1G083RhkCj0Wi6OdoQaDQaTTdHGwKNposjIjNFJLuzx6HpumhDoOlURGSviHhExC0iuSLygoi4OntcTSEi/UREiYitnY4XKSIlInJqmG0Pi8j89jiPRtMY2hBojgXOVUq5gBOAicAdrdlZDLrMd7m+AVFKVQFvAj+u188KXA68ePRGp+mOdJl/Hs3xj1IqB/gYGCUiCSLyoYjki0ix+T4j0FdEFovIvSLyHVAJDBCRn4jIVhEpF5HdIvLzkP4zRSRbRH4nInkiclBELhCRs0Vkh4gUicgfQvpbROQ2EdklIoUi8paIJJqbvzFfS8yZzDRzn2vM8xeLyKci0jfkeEpEbhSRTCAzzOW/CPxARKJC2uZg/I9+3NS11cc816CQzy+IyD0hn+eKyDpzFrJURMaEbPu9iOSY59kuIqc1dh7N8YM2BJpjBhHpDZwNfI/x3Xwe6Av0ATzA4/V2uQq4DogB9gF5wFwgFvgJ8LCInBDSPw2IBNKBPwP/Ba4EJgAnAX8Skf5m318CFwCnAL2AYuAJc9vJ5mu8UsqllFomIucDfwAuAlKAJcDr9cZ7ATAFGFH/2pVSS4GD5v6h1/eaUsrbgmtrESIyHngO+DmQBDwFLBSRCBEZCtwETFJKxWAYor2tPYemC6KU0j/6p9N+MG40bqAE42b+JOAM028cUBzyeTFwVzPHfg+4xXw/E8OYWM3PMYACpoT0XwNcYL7fCpwWsq0nUAvYgH7mvraQ7R8DPw35bMGYqfQ1Pyvg1GbGewfwmfk+1tx/fAuvLTtkmwIGhXx+AbjHfP9v4O56x9qOYfAGYRic0wF7Z3839M/R+9EzAs2xwAVKqXilVF+l1A1KKY+IRInIUyKyT0TKMNwx8abfPMD+0IOIyFkistx085RgzC6SQ7oUKqV85nuP+Zobst0DBBaq+wLvmu6TEgzD4AN6NHINfYFHQ/oXAYIx+wg73jC8DMwSkV7AxcAupdT3Lby2ltIX+E1gnOaxegO9lFI7gV8BfwXyROQNcyya4xxtCDTHKr8BhmI8scdy2B0jIX2C0rkiEgEsAB4Ceiil4oGP6vVvDfuBs0wDFfiJVMY6RjjJ3v3Az+v1dyrD5dNgvOFQSu3DcCldieEWevEIr60SCF1rSKs3znvrjTNKKfW6OYbXlFIzMAyGAh5oasya4wNtCDTHKjEYT+gl5iLtX5rp7wAigHzAKyJnAbPbcP7/APcGFnxFJMVcB8A8hx8YUK//7SIy0uwfJyKXHMF5X8Tw008HXjXbWntt64AfiYhVRM7EcPsE+C9wvYhMMaOtokXkHBGJEZGhInKqaXiqMH7//iO4Bk0XQxsCzbHKI4ATKACWA5801VkpVQ7cDLyFsbD7I2BhG87/qLn/ZyJSbo5hinmuSuBe4DvTvTJVKfUuxtPzG6YraxNw1hGcdwGQCHyhlDp4hNd2C3AuxrrLFRjrCZjHWg1ci7HwXgzsBK42N0cA92P8zg8BqcDtR3ANmi6GKKUL02g0Gk13Rs8INBqNppujDYFGo9F0c7Qh0Gg0mm6ONgQajUbTzWkX9cSjTXJysurXr19nD0Oj0Wi6FGvWrClQSqXUb++ShqBfv36sXr26s4eh0Wg0XQoR2ReuXbuGNBqNppujDYFGo9F0c7Qh0Gg0mm6ONgQajUbTzdGGQKPRaLo52hBoNBpNN0cbAo1Go+nmaEOg0WhaT2kObP+4s0ehaSe0IdBoNK1n1TPwxhXg13Vrjge0IdBoNK2nqhSUD2rKO3skmnZAGwKNRtN6qk0DUFXWuePQtAvtYghE5EwR2S4iO0XktjDbTxaRtSLiFZGL623zicg686ctpQU1Gs3RosZtvFZrQ3A80GbRORGxAk8AZwDZwCoRWaiU2hLSLQujLuqtYQ7hUUqNa+s4NBrNUUTPCI4r2mNGMBnYqZTarZSqAd4Azg/toJTaq5TaAOiVJY3mOGBPTQkLXNF6RnCc0B6GIB3YH/I522xrKZEislpElovIBY11EpHrzH6r8/Pzj3CoGo2mPXiXcv6akoTPU9LZQ9G0A8fCYnFfpdRE4EfAIyIyMFwnpdTTSqmJSqmJKSkN6ipoNJqjSLm/FoAKT0Enj0TTHrSHIcgBeod8zjDbWoRSKsd83Q0sBsa3w5g0Gk0H4lZe47WysJNHomkP2sMQrAIGi0h/EXEAlwEtiv4RkQQRiTDfJwPTgS1N76XRaDoVvw+3udxXXl3cyYPRtAdtNgRKKS9wE/ApsBV4Sym1WUTuEpHzAERkkohkA5cAT4nIZnP34cBqEVkPfAXcXy/aSKPRHGvUuHFbjFtHRVVJ545F0y60S81ipdRHwEf12v4c8n4Vhsuo/n5LgdHtMQaNRnOUqHbjtggA7hodNXQ8cCwsFms0mq5EdXlwRlAeSCzTdGm0IdBoNK2jxk2FmK4hb2UnD0bTHmhDoNFoWoW/qiToGirXhuC4QBsCjUbTKjyeYpSYawS+6k4ejaY90IZAo9G0ivLKw0lkbn9NJ45E015oQ6DRaFpFRXVJ8L1beXVxmuMAbQg0Gk2rKK86nETmFjksSa3psmhDoNFoWkWFqThqE4sRRqoVSLs82hBoNJpWUV5r1CLoYY81DIGuSdDl0YZAo9G0igrTFZQWmUS5nhEcF2hDoNFoWoXbzB1Ic6ZQYRE9IzgO0IZAo9G0CrfPA0BqVA/cFguqqrSTR6RpK9oQaDSaVuH2VhONhVhnEl4Rqjy6JkFXRxsCjUbTKtz+GlxiJcaZBECFNgRdHm0INBpNq3ArLy6xE+1MBKDco4vTdHW0IdBoNK3CrXy4rBHEOGKMzyGZxpquiTYEGo2m5fj9uMWPyxqJy+ECdHGa4wFtCDQaTcupceMWCy6bE5c9YAi0xERXRxsCjUbTcsx6xS579OEZQa02BF0dbQg0Gk3LqS7HbRFcdtfhGYHX08mD0rQVbQg0Gk2LqfWUUGWx4IqIJdoeDYDbV9XJo9K0lXYxBCJypohsF5GdInJbmO0ni8haEfGKyMX1ts0TkUzzZ157jEej0XQMFZX5ALgi4rBZbDixUq6rlHV52mwIRMQKPAGcBYwALheREfW6ZQFXA6/V2zcR+AswBZgM/EVEEto6Jo1G0zG4q4zkMVdEPAAxFjsVygtKdeKoNG2lPWYEk4GdSqndSqka4A3g/NAOSqm9SqkNQP1SRnOAz5VSRUqpYuBz4Mx2GJNGo+kA3J4iAFxmMlm0NYJyi0BNRWcOS9NG2sMQpAP7Qz5nm23tuq+IXCciq0VkdX5+/hENVKPRtA13VQkALmcyADE2J26LaCnqLk6XWSxWSj2tlJqolJqYkpLS2cPRaDofnxdqKo/qKd3VhtKoK8owBC5bNBWii9N0ddrDEOQAvUM+Z5htHb2vRtO9WfIPeOrko3rKQBZx0DXkcOniNMcB7WEIVgGDRaS/iDiAy4CFLdz3U2C2iCSYi8SzzTaNRtMM+bnrWe/eB77ao3bOQBaxy9QZinHEGK4hPSPo0rTZECilvMBNGDfwrcBbSqnNInKXiJwHICKTRCQbuAR4SkQ2m/sWAXdjGJNVwF1mm0ajaYanK3dzQ48UqCg4aud01xqLwoFkMldEvFnAXhen6crY2uMgSqmPgI/qtf055P0qDLdPuH2fA55rj3FoNF2a71+BvtMhsX+Luuf6KimzW6kqyyYytmcHD87A7fNgUxBhjQDAFRmPx2LB6ylpn5uJplPoMovFnUL5IVh4M1SXd/ZINMc73hp4/0ZY3fJnokJluISKind31Kga4PZV4cKCiADgChanOXqzEk37ow1BU3z7MKx9EfZ809kj0RzvBOr+lh9qWf9qNwXGvZjCsqyOGVMY3L4aosUa/BwwBOUe7dHtymhD0BieElj7svE+d0unDkXTDQgYAnfLDIFy51JgNW7IReUHOmpUDahQtcTIYSdQQIG0Qhen6dJoQ9AYa16A2gpwxEDe5s4ejeZ4p6qU5ZERFLkPtqi7u3QfNRZjSlDoyevIkdWhXHmJtjiCnwOLxuVVerG4K6MNQTi8NbDiKeh/svGTqw2BpmPxegr5RVoqr6iW3VALQtYFCo9i8fgK5SfGGhn8HCxXWavX0boy2hCEY8t7UH4Apv0SeoyAwl1Qq6V2NR1HeUUuXhFyaVm2cEHZYWWWoqNVKtLvp1wg2nbYEASlqLXWUJdGG4L6KAVL/wXJQ2HQ6dBjJCgfFGzv7JFpjmPKKwz9rDybtUXrBIUVhgvJAhR6j5LMRG0FFRbBZd78IWRG4NWGoCujDUF99i6BQxtg2g1gsUDqSKNdu4eOHyqL4MP/g+pjp8RiueneybdaWxQ5VFBprAv0sUZR5D869QBUVXmwTGWAYJUyXZymS6MNQX2WPQFRyTDmMrYXbeeHy+8g3+7UhuB4YtuHsPpZOLC2s0cSpKyqGIA8q61FhqCwqgSbgv6ORArFb6xrdTDVnkK8IrgcscG2CGsENgS3Lk7TpdGGIJT8HbDjE5h8Ldgjeez7x9hatJV1qf0gT4eQHjdkrzZej7JyZ1OUmxIN5VYLntL9zfSGgtpyEsVGcmQiRVYrVHS8NLvbnIWEGgIRwWWx4/bX6uI0XRhtCEJZ/gRYI2DiT9lUsIlvso1Esj0xyXpGcDyRs8Z4rT12/NrlIQu++aV7m+1f4K8i2RpJYlQyxRYL3vKWhZ22BXelWZ0sMr5Ou8tiFqepPXYMq6Z1aEMQoKIA1r8BYy8DVwr/Xv9v4iPiSYpMYrfDDu5cqDh6YXqaDqLaDXlb2Gm3H1szghCjlNdcLoHfT6HykmyPISm6J0qEkpJ9HTxCcJvrGDGmBHUAly2SCouuSdCV0YYgwKpnwVsF024MzgbmjZzH0MSh7AksxunEsq7Pge9ZFmHnwoyebCvv+JtnSykLifzJ9+Q23dlTTIHVQnJEAomxRjmPwhbMItqK21zHiDZlJQK4bFHGjEBrcnVZtCEAI0dg1X9h8GxIGRqcDVw+7HL6x/VnT1U+CrR76HggZzWrnEYcfEHVsaOPU+6rwoaRKZxnloNsDH/5AYqsVpKikkmK7wdAobvjZSYC1clinHUrBLrsLtyii9N0ZbQhANg031hsm3YTG/M3BmcD0fZo+sf2x+OrItel1wmOC7JXs8EVD4DnWFoj8NfQwxJBJBbym4nJLynZg0+EpOieJLl6AVBU2cwsoj3GaK5jREcn12l3OVyma0jLTHRVtCEA2PoBxPeF/ifXmQ0A9I8ztOH3JPfTkUNdHaXwZa9mo8148vYcQ4ub5cpLrCWCFFs0ec1kFxeU7AUgObYPiZGGv/5oyExU1BiunxhnPUMQEWe6hvSMoKuiDUFtlSEzPXg2Gws2sSRnSXA2AJASadTT2ROTAnlbwe/vzNFq2kJZDjtrCqnE+Bt6jlZGbnN4qykXRazNSWpEvJFU1kR2cUG5EV6aHN+fWEcsNnV0ZCYCC9pRjug67TERCVRYLCiPnhF0VbQhyFpqhL0NPqPBbODjjQc5/cHvcVqj2eNwGP2K93TygDVHTPZq1kVEBD96vMdINmxVGeUWCzH2aFKdyYbMRHnjrp7CCmNbclwfRIREsVN4FCQeKryVOBXYLfY67dHOBHwieI6hNRdN69CGIHMRWCPYGJNYZzZQWePlrg+34PNDJGnsUYHIIe0e6rJkr2K900liRAIAnmNFFqGqlDKLhRh7DCmuXuRbraiyxhd/C8xqYMmmiybJ6qToKFyL21tFtJIG7TFmFJFbF6fpsmhDsPNz6DeDf29+vs5s4N+Ld3GwtIpBqS5KyxLYU5UPiC5S05XJWcP6KBfjUscTocBzrMgiVJUaM4KIWFJj++KxWHCXNZ5dXFhTilMJUbYoABIdLlNmomOvx+2vxiUNbxmBTONAeKmm69G9DUHxXijYwYaMMXVmA1mFlTz1zW4uGNeLX58+BE9FEnmefCoS+0Hups4eteZI8NVSdGg9WeJjbOpYnGKh0lfb2aMCoLayEI/FQkxkAilxfYCms4sLaitItNiDdYOTIhKOisyE21+DSxqWqA9UKXPrqKEuS7sYAhE5U0S2i8hOEbktzPYIEXnT3L5CRPqZ7f1ExCMi68yf/7THeFpM5udUifCXopUkRiYGZwN3/28LNotw+9nDOW14KnZ/DwD2JvfXrqGuSu5mNlgNLZyxKWNxYsWjjg1D4DZDP2MiE0mJSgWazi4u8FeTbI0Kfk5yJlNotaJaWu/4SMepfLhCqpMFCCqQ1uqooa5Kmw2BiFiBJ4CzgBHA5SIyol63nwLFSqlBwMPAAyHbdimlxpk/17d1PK1i5yIe6Nmbne793DfjPqLt0Xy9I5/Pt+Tyy1MH0yM2kki7lel9jMvZ6UqCot3HlDSBpoXkrGZ9pAObWBmZNBKnWPH4jw1DEKhFEBuVTKppCPIbKz/praZQ/CSHCL8lRadRK0J5C8Tq2oJb+XBZIxq0B2YE5TU6s7ir0h4zgsnATqXUbqVUDfAGcH69PucDL5rv5wOnSWBe21nUVvHJoeXMj4BrRl3D9PTp1Hj93PnBZvolRXHNjH7BrpeMHYdSFlbVCig/5G/rvHFrjoxsY31gaOIwIm2ROC32YBhpZxOoRRATnUqKmbWb11gxeHceBVYryZEJwaaAzERRB8tMuEXhCilTGSAwI6g4VsJxNa2mPQxBOhD6KJJttoXto5TyAqVAQLCkv4h8LyJfi8hJjZ1ERK4TkdUisjo/v+2+0OztC7kzwcUYVx9uGn8TAC8t28vu/Ar+fO4IImzWYN+ZQ3si3iRWu80nniNwDyml8Pu1TG9n4c1eySaHjbEpYwFwWux4lK+TR2UQqEUQ40wmyh5FjNjIayQctLbsACVWK0lRPYJtSXF9ASh053TcIJXCLYauUH2CMwKvp+POr+lQOnux+CDQRyk1Hvg/4DURiQ3XUSn1tFJqolJqYkpKSrguLabWX8vv1v8LAf4+61HsFjt55VU8siiTWUNTOHVYjzr97VYLPZx9yK4uQNmcjUcOKQXv3wjfPNhg050fbOGSp5a1adyaI8RTTGb5Pjyow4bA6sCDOiY09AO1CGIijK9+ii2afHxQ2/DGWlRiFK1PiukVbEty9TS2VTTiTmoH/NXlVNSrThYg2mbWLT5WorA0raY9DEEO0Dvkc4bZFraPiNiAOKBQKVWtlCoEUEqtAXYBQ9phTE3yr7X/YqO3hL9ae5GeOAiAv3+ynWqvjz+fOzLsPuPShoC9gEJX/8Yjhza8Cd+/Al/eA8v/HWxWSvHJpkOs2VdMfrn+Zznq5KxlvZlINjY1YAgi8Ih0eMhlSwho+MSafv+UiHjyGilZWVBqKKYmx/YNth2WmSjosDFWVBqz8MDTfyhWi5UoseH21xwThlXTetrDEKwCBotIfxFxAJcBC+v1WQjMM99fDHyplFIikmIuNiMiA4DBwO52GFOjfJvzLc9vfp4flpUze8hFAKzbX8L8Ndn8dMYA+ic3fOIBOLH3cMTi4xt/j/CuIU8JfHYHpE+AYXPhk9thy/sA7C2s5FCZkfCzfLeuaXDUyV7N+sgIkiOT6BVtPEk7rU48x0gxlTJTuiFQCD7VmdJo7eLCcuMZKzlhQLAtwVwvKKrpuPDNiopAdbI4ALw+P8UVh8tjuiwO3IIh5a7pcrTZEJg+/5uAT4GtwFtKqc0icpeInGd2exZIEpGdGC6gQIjpycAGEVmHsYh8vVKqw9IT8yvz+eO3f2RwRBK/LSoxZKeB/36zm4QoOzedOqjRfQfEG+Jzi6qdRry2u940/Kt7obIQzvkn/OAZ6D0ZFlwL+5aybJdx87dZhGXaEBx9clazPsrF2NRxwdj7KJvTmBHUdL4Cabm3EgsEE8RSXL3Is1lRYaqOFZihpsnRacE2m8VGPFYKa90dN0ZzRhAdaRiCx7/ayan/WEytz1hwj7FG4tbFabosDbNDjgCl1EfAR/Xa/hzyvgq4JMx+C4AF7TGG5lBKccd3d+DxenhI9SQyvh8kDaS4oobPt+RyxdQ+uCIa/3UEVEh32uxQjSFJ7TJC/TiwDlY9AxN/Cr3GGW2XvwHPngGvX87utEdJjXExslcsy3dpQ3BUUYrCA2vYnxrNJeb6AIDTHoXHYjk2ZgS+KmKs1qCRSo3ri1eEkpK9JNTrW2guLCfVKw5jyEx0XGZvRSCyyXRDfbLpEMWVtewtqGBwjxiibU7cAQXSmB5NHUpzDNLZi8VHDRHh2tHXcufkOxiwdwUMPgOA99flUOPzc8mE3k3uHxcRR2JkIu4YM+o14B7y++F/v4GoJDj1jsM7RCXClQtQVgfX7L2VOX1h2sAkdhdUkFump89HRN42o6RoayjazQZl3OzH1jMEtSJ4j4En2HJ/DbEhQm4pZhRQXllWg74FNaXEYCGiXjx/oj2aQvEZarodMcZAdbLIBA6Weth2yIig23LQ+P3F2KP1jKAL020MAcDEtImcJdHg9cAgwxC8vSabkb1iGdErbLBSHfrH9ccZW0mBisWTvcFo/P5lyFkNZ9wNzvhg39LqUjb7K3h96i95P9aLr+K3VFoXAXqd4Ej53bsXseCTG1u3U84a1kcYiWQjkg7nOTrN2HdPY/H6R5Fy5SUm5MaeaoaG5oepOlbg85BkaZjUlRSRQKHVCh0UOVRhVk2LiUrh6+2Hw7cDBsHliDGrlGmZia5Iu7iGuhSZn4MtEvrNYMuBMjYfKOOv59ZPhA7PgLgB7Cj6jG3+3gzL2oCzsggW/RX6nGgUvQf+tuJvfLDrA8prQ7Isk2KwqVpid/6H2Mj7WbarkPPH1U+10DRFrTuXTyMs+Cuy+UFrdsxezXpnFMPMRLIATnNh1lNVQkz7DrV1eKspF4ixOoNNgeziPE+9fBmlKPDXkGyr6xYCQ2aiyGoFdz7E92n3YQZCXKOjUvhqex494yKJc9rZZs4IXI5YPSPownSrGQEQVBvFEcXba/bjsFpafFPuH9ef8tpS9rj6EVO+Ez7/k1Ge75yHQITdJbt5bdtrjE0dy60Tb+WRmY8w3noXroP3c6NzAEV4mdA/+thfMK4qg7fmGXIaxwjZOSvxi1Dsa13Skjd7JZsdDsamjqvT7owwDUFnP8FWlVFuEWJD4vMD8tJ59cdWXUahVUh2xDc4TGJ0D9wWC9Vl2R0yzArzwSYiMpnvdhYyc2gKw3vGsvWgOSOIjD+8RqDpcnQvQ1C0Bwp3wqAzqPH6eX/dAU4fkUpCdEMhrXAEFoyrMnoTQY2RMzD1F9DDyD2Ynzkfm8XGPdPvYd7IeczqfSqb9riYNiCdjLh+AIzqWcq+wkoOlBzDWZg7F8GW92D18509kiBZuesAKGqNUFxtFTuKtuERVWd9ACDKjNmv7HRDYEpQOw7PSxxWBwliJ99bLwqoPJdCqzVoKEJJijEq6RWZeQbtTXmNG1GKLbmCu9rLzKGpDO8Zw6GyKooranBFJlJlsVDr0VLUXZHuZQh2Gj56Bp/Bl9tyKaqoaXaROJSAIbBmmIU4HCkw04iErfZV88GuD5jVe1YwomNHXjlFFTVMG5BERqKRJ9fTaSwALjuWo4d2LzZet7x/zCQIZRVnAlCEv+VjOrSR9XZDKqS+IXBGxgPgqem4kMsWETQEcXWaU+wu8uplF3tK9+O2WEiKTm1wmERTvrqwvINmBN5KXAq+ySzEbhWmD0pmWJphTLcdKsflNKKJKo6T4jSHSqvYlNN91ju6lyHI/AwS+kPSQN5enU1qTAQnDW74dNUYPaN7EmGNoMip2BYxmj/5rqXGakzpv9j3BSXVJVw85OJg/8DNftrAJDJSxgDgrd1HfJT92HYP7fkabE4o2QeHNnT2aADYZ+rolFgEX0v90AfWsj4ygpTIRHpG96yzyRlh3Hg9nayYWVtZYNYiiK/TnhIRT76tblJZYVBeouHDS5KZV1BU0XiJy7ZQ7vXgUrB4ez6T+iXiirAxrKcxi9l2qAyXKY9RfpwUp7nzg83Me24l6hh5EOpoupchOOlWmHMfeeVVLN6Rz0UnZGCztvxXYBEL/WL7sac8iwMXLuDdilF8stn4R52fOZ90VzpTe04N9l+2q5DeiU4yEqKISx2Jy+8npzyLKf0Tj90ZQfFe42f6zSDWYHZ0Z5NVbTxp+kUoLWlh3eiSLNZHRjI2dTz1xW6dZjaup7ZzE8rKzSifmHp5AanOVLOI/eEbe0FZoGh9vwbH6WiZCbevimgsbM8tZ+ZQQ+srxRVBUrSDrQfLiLEbRsF9DERhtRWfX/HdzgIKK2o4WNo9Qr27lyHoMwWGnc27a3Pw+RWXTMxo9SH6x/VnT+keZg5JpW9SFC8u3cve0r2sOrSKHwz+ARazlJ/fr1ixp4hpA4x/cIlKJN3rJ6cyl2kDksgp8bC/qPOTmRqw+2vjdeRFxqL6seAeUoosVU2EOYyiFhqCwrIcsm1WxqaMa7DN6TBmcm02BGUH4NDGI949kLEbU8/vn+LqRYHViq/ssGxXoRlOGtAZqvb6KHQbWkkBd2RhB92IK/w1RCrjuz1zqOGaEhGG94xl26Fyos3fp7uz11zagc0HSimr8gKw9WD3WPzuXoYAI8P47TXZnNAnnoEpDQW0mqN/XH9y3DnUqhqumtqXNfuKeer717GKlQsGXRDst+VgGaWeWqYNNJ/0RMiwRJJdU8q0gcY/fWe7hxauP8DYOz8jJ3Thes/X4EqDlKEw4jxjcT1va+cNEqgpy+Gg1cJIm+F+KC5vWQGW9ZVGv4DQXChOmxGu6WmrdPLHv4NXGyTNt5hALYLYen7/HnH98ItQWLI32BZ42k+OMp7I7/xgC1P/9gWPLsrESgRRWDpMZqLc78Xms5Ie72Rw6uH/m2FpMWw/VE6ULVClrJPXXNqBpSGz9S0HtCE4Llm3v4SdeW4umdjyReJQ+sf1x6/8ZJVlccnE3jjtis/2fcgpGaeQEnVYHjuQNDZtwOEnvQxHHNmqmsGp0SRGOzpVbsJT4+Pe/22h1FPLs0vMJ2y/35gRDDgFRGDYuYDA1voagkeXQOjouIShABQ2UcYxlO3Vhr96eOLwBtsCuj5tMgR+P+z9DsoPtj7j2SS0FkEoKab7Jz8kCqigqghRhshcqaeWd9Zmkxjt4OFFOzj3X98SJxEU+jrGleHGB7UWThmaUsfNNqxnLNVePyVuY1HefQxIdrSV73YWMDjVRb+kqGDm9PFOtzMEb63OJtJuYe6Yns13DkMgcmhP6R7inHamjj5IDWXM6XNBnX7LdhXSPzmatLjDSUzpUT2oFuPJbuqARJbtLuy0xahnluwmt6yaUemxvLEqi9LKWkM2o7IABsw0OsX0gD7TOn2dICvfWLAe3+dUoOW6+/m+ChLFXieRLEBgRlDZlhtnwQ4IRMkc4aypfi2CAIHs4ryKw0avoNZNglixWWy8uzabqlo/z86bxDM/nkippxZvpZUi8eOpaP8FcLfyY/HZmTmkbi2QYWnG2kBOkSE+5+7iVcqqvT5W7S1i+qBkM09CG4LjDk+Njw/XH+CsUT2JibQ3v0MY+pr+2T2lxlN0TdQy/LXx7N5/uFCI1+dn5Z4ipg6ouwAYyCXIKdzGtAFJHCytYl/h0f/HyS+v5j9f72LOyB48ePFYKmt8vLJi3+Gw0f6nHO484nzDQBRkHvVxBthXZJx7zIDZiFIUV7UgRNFbTYHykmQLLytutVhxKPB4a8JubxFZS3k7Jpp7khKO3BCYtQhC8wiA4OwyP2Txt8BfRZI1EqUUr67IYmxGHKPS4zh9RA8++7+TSYyIp9Bq4ep//Y+lu9p30bhCwOp3MH1Q3ZnL4B4urBYhK980BF28OM26rBKqav1MG5jEiJ6x7C2sxF3t7exhdTjdyhB8uvkQ5dVeLpnQ+kXiAE6bk17RvdhTtof95fvZULiKNDmZ15Zn4zUleTcfKKO82nt4fcAkI3EYAPvz1ge3NbZOUOgp5LrPruNgC90greGRRTuo9vr5/ZnDGN4zllOGpPD8d3vx7VoMSYMhLiTTevi5xmsnzgqyKnKIVZAYnUq8kpbp7lfkU2C1khIR32gXJxY8/rYYguW8HZfAOzEuanOPbMG43FysjnXUnREkRiZiAfJqSowGn5dC5SXZHsPqfcVk5rn50ZTDUhKxkXZGp2VQZLWSSAlXPbuSnXntMzOo9dZQbRFiHNFE11PojbBZGZgSTWZuFXYslB/B7zOvvIqq2laUDfUUG8mc2/4HB743JOH97VN/+rtdhVgEpg5ICuqPbT90/M8KupUheHvNfjISnA2e1FtLIHLoncx3sIiFa8dfxoHSKhZtNVwWgZv71AGJdfbr1WMMohTZxTsZmOIiJSaiUQG6lYdWsuzgMj7Z+0mbxlqfzNxy3li1nyun9mWAuVj+85MHUOquwL/3W2N9IJS4dMiY1KnrBPtqSugrhnsnQWwUtyTSx51Lgc1KcmTjf2unWPD4W5GpXP8U+5ax3W6hVoT9+Y1UrWuGcm8lVg67qgLYLDaSLBHkB1wtlQVGVnFEAq8u30dMhI1zx/aqs09idColFgv3zk5FMNyg7cHOXGPRvYcrPuz2YWmG1ITLYqcCf6uqvu0vquS0f3zd8ph9vw/euMIoCfvGj+DpmfDQYLi3Bzwyhqpnzw1b0KelLN1ZwOj0OOKcdob3NAxBd1gw7laG4E9zR3DvhaOxWKT5zk0QMATv7XyPk9JP4uKxI0mPd/Li0r2AsT4wKNVFakxd33RE0iBSfD5yyvcjIkwdkMSyXeHXCTLNTNqlB5a2aaz1uf/jbUQ5rNx82uBg27SBSVyUegi7z4O//ykNdxp+Hhxcb0h0HG2UIkvV0Me8oSdaIij0N+/XV2WHKLBa6xRwqY9TbHjUEU77S7NZV5NH4Dk0s2zfEYXZlvmqiMHaIM8BIMV2OLtYlR+iwGohxpHIR5sOceEJ6UQ56j6dJ8Vk4BdB1Rxg1rBU3lmbEywc0xa+3bYDgN4J4Y3q8J6x5JR4iLZEUN4K4blan5+b3/ieyhofK/YU8faaFhiu7x6Ffd/B2Q/BtV/Bpa/CWQ/C1F+QGzeGyP3fkPXl0y2+tlAqqr2s218SjOrrGRdJfJS9WywYdytDMCzNcIO0lf5x/fF4PRR4Crh4yMXYrBaunNqXZbsL2XyglFV7D+cP1CEyjgwfZJt+32kDksgrr2Z3QcMn3IAhWJu7lqp2Kv+3dGcBX2zL46ZZg0gM0VcSEa7plYVPCV9VD2244wiz0NzWD9plHK2humQfh6wW+ppaOol2F0UtuHmXle6jVoTkmMbdgE6LjcojNQT7lrEmMgKbWLEg7BQvlB6+kXl9fjJzm3fNlPtriLGEX69KjUgwahe7c3GXZlFtsVBUEUON11/HLRQgKDNRls0PJ/amwF3N4u35Dfq1lvV7jAeAXrHhs/ADGcYOMauUtVB47tFFmXyfVcLDl45jUr8E7vtoazAvIiwHvjcqAY64ACb9DNJPgOFzYcp1cMZd/MX2a1b5h+Dc9s4RGeWVe4vw+hXTB5m5PyIMT4tly8HOzT4/GnQrQ9BeBCKHUp2pzEifAcClk3rjsFm4bcFGKmt8DdYHAmRYnWR7jS9WcJ0gTBhpZkkmSZFJ1PhrWJu7ts1j9vsV9360lfR4J/NO7Ndg+5CK1eywDuTJ5WFcVQn9oOfYTlknyM5ZiRKhjxk6muCIpVgU+Jq+geebWbgp5gJ9OJwWOx51hE/MWUtZ64xiRNII+jhTyXTY6ywY3/XhFuY88g1ZzQQDlCsvsdaGUU1gyFEHZCYKTHmJTQcjmNA3IajzE0qSKV9dVHGQmUNTSHY5eHt1y3IuGqPG62d3npHU5qongxFguDkWiz+CCosYirzNsHRXAU8s3smlE3tz3the3HfhaCqqvdz30bZGBlJplH6NToW5DxvhzSEUVdTwxbZc3vdNJ8Wz54iS/JbtKsRhtTCx72GX7ohesWw/VIbPf3xLTWhDcAQMiBuAVaxcNOQibBZjep4Y7eD8sb3YaApVNbYOkRGZSJ7yUuOroV9SFD1iIxosGFfUVpDjzuGiwRdht9jbxT303rocNh8o43dnDiXSFGILUu1Gclbj7Xsya/YVs3pvmKicEecbBXhKO0bUrDH25a0HoE/aOACSIhMptVqpdTetqVNQYWThJrkaL5votDjwcGSGoHrfUjZGODihxwQGJw1jp8MerFq3am8RLy3bh1/B/zY2sdjvraasXi2CUFJi0im2WqkpzabAlJfeUxLPFWFmAxCSXewpwG61cNEJGXy5LY+Cpp6ym2H13iJEmTUHnOG/0z1iI4iPsuPzma6hZmYERRU1/PrNdfRPjuYv5xm1QAb3iOHnJw9kwdrs8BFPn/3RSG688D9G9b96vPd9DrU+hW/Y+dQqKxVrXm/llRr5A+P7xON0HP7/GNEzlqpaP3vCzNqPJ7QhOAKSnEm8OfdNrhtzXZ32wJP2sLSYOq6XUDKie6EEDrhzEBGmDUhiRb18goBbaMzyZzkhfgjLDi5r03iran08+Ol2xmbEce6YXg077FsKfi+Dp55DQpSd/3wdpg7B8PON16PsHsoq2QVAn7RJACSaT70lJU3XSsg3pRtSnI27Ap3WCKOAva/5BeON2aWHn+4ri9hYvodaFCeknsCgpBFk2WxU5W6iqtbHbQs2kB7vZHjPWD5qyhBUlRnKo/bwIa6pZsnKgpLdFJpicg5bKmePDp8Dk2SuoxSZ1cQumZCB16947/ucsP1bwlfb84iwGdcdUBitT8CFUlUT0WxxGqUUv5u/geKKWh67bHyddY6bTh1E36Qo7nh3U90oou0fw+rn4MSbGgYzmLy9JpvR6XH8aNYJfO0fg2XTAmNhuYUUV9Sw5WBZg/DY4ILxcb5O0C6GQETOFJHtIrJTRG4Lsz1CRN40t68QkX4h224327eLyJz2GM/RYGjiUOz1fLuj0uO4fHJvrpzat9H9MuIGAJCdZ0SZnDq8BwXuGj7dfDjSIbPEMASDi/YzraqGHcU7gje2Wp+f/3trHb9+cx3//WY332YWNPCrKqXIKqzkfxsOcv/H27js6eUcLK3iD2cPD79QvudrsEYQOWA6V03rx6KtuezMqycVkDwIUkfCllZGD/lqofLIpYn3VRwkXkGcWQY0wWXcBJvT3Q9o7oTT7g/gtEXisQjUNP20t2pvET/491IufPI7due7IWs5ayOMcpEn9DiBQfGDUCLsLtjME1/tZFd+BfddNJqLxqezMaeUfYWNHL+q1ChKY+YQfLktt07IZ4rpgswr28+hCuPvf+7IIQ1ndCaxjlhsCIVmEZnBPWIY2zuet1bvP6LExRqvn4XrD9AzzrihusLIXwcY1jOGkkqHWa6y8Zvmy8v3sWhrLredNYxR6XWltyPtVu4+fxS7Cyr492LjAQB3Hrx/E6SNhlP/FPaYm3JK2XqwjB9OzGBkr1gW2U7BWZVrLCq3kOW7C1GK4PpAgEGpLuxWOe4jh9psCETECjwBnAWMAC4Xkfq1H38KFCulBgEPAw+Y+44ALgNGAmcCT5rH67L87aIxTRuCZONXk11guBHOHpXGgJRo/vHZjqAfMrM4k2ix0svrY1qW4RpZfnA5AG+szOKdtTksyczn3o+2cuWzK5hwzyKm3LeIq59fyY/+u5yxd37GyQ9+xY2vreWZJbup9fm57axhTGksbHb3YkOQz+5k3rS+RNgs/PebME/cI86DrGVQ3gqp468fgMcnNevTb4ysmhL6WAw5iOW7C7HYDFdPUTOVuApqy4nEQnQjT9sATqvTmBE0IYuwr7CC615aTa94w49/1bMrqchcwlqnk0FxA4mLiGNQwiAAdlTk8PTiHVw0Pp1ThqRw1mgjYqlR91CwFkEsO/PKueaF1cx5ZAl3frCZUk8tqdFm7eKKg2RXFGFRMG9q42VVRYREi6OOzMQPJ2awI9fNhuzWi8F9uOEAuWXVpJsTAVdU44ZgeFosHm8UFRZBecKfa+vBMu7531ZOHZbKT6b3C9vn5CEpnD+uF/9evIudueVGmGiNGy56BmwNazUDzF+TjcNm4byx6VgsQvXAOVQQidrwVouv9btdBUQ7rIzJiDcaDm2EHZ/isFkYnBrTIRnGpZW1Rkb/MUB7zAgmAzuVUruVUjXAG8D59fqcD7xovp8PnCZGvNz5wBtKqWql1B5gp3m845bk1FFE+P3kmC4Pm9XCb84YSmaem4XrjSl8ZnEmg/wWxB7FsPICEmzRLDuwjLKqWh5elMmU/oms+uPprP3TGbz6syn88ezhTB+YzKHSKsqrvJwzxlh8W3jTdDbfNYf/3XwS158yMPyA3PmQuykoK5HkiuCSiRm8+30OeWX1opVGnA8o2NZy99C3mR/wlyg/6kjqGvh97KOWvpFJvL8uh8ueXs4Lq42EpaKKJmLFlSLfV0WyNTJsWGaAKHsUHrEYC5FhKPXUcs0Lq1DA8z+ZzIvXTKbUU8vudYv4PjKSCWkTAegT0weHWNllFUZGFvKnucbNOiMhivF94vnfhvCGoKaygCqLhZjIBF5etg+H1cLFJ2TwwtK9zHpoMUu2Gdea5ymkyFtBrN/KkB4NF4lDSbK5KBJ/cJZz7theRNgsvL2mdYvGSimeWbKHwakunBG12JUiIoxvPsCwnjHU+GLwi1AZJvO7qtbHL1//njinnQcvHtPk3+WOc0YQabew+NX7jRoiZ9wNqcPC9q32+nhvXQ6zR/QgLsqYoU8dmsEnvon4N78PtS2LuFu6s5DJ/ROxWy1GAatnZ8NrP4TtHzO8Z2y7u4Z257s59R+L+fkrq9v1uEdKexSvTwdCv2XZwJTG+iilvCJSCiSZ7cvr7XtcV3WXhL6ke31km5LCAGeNSmNEz1ge/jyTc0b3JLM4kzMqymHcj7Bsfo+pysHSA0t58qudFFXU8MdzhhtPf9FGyn99v2ar2PuN8dp/ZrDpZzMG8OqKLF5Zvo//mx0STpoyDBIHwI7PjPC95qgoZIGvgEUxLn6Q+QFj0k9o1dCqinaRa7MSbenBrW+vxxVh49u9dmKGQHH9wu6hVJdRaIFke9Nl6Z32aGosgq+mnPrT0FqfnxteXUNWUSWv/HQK/ZONmcUzPxqBf34WlZLKqKRxgJH8lWhJJdPh5vaJqk7p03NG9+Se/21lb0EF/ZLrzk7KTb9/hD2e+WuymTumJw9cPIYfn9iXOxdu4Z6Fe4kfBlmVRbittSRZmzYCAImOWAqtFsOlktif2Eg7Z45KY+G6A+YNtmUT7mW7CtlysIz7LxpNZlYFLr8f7FGN9h/SIwaljEVvt6eI+vOwx7/cyc48Ny9dM5kkV/gn+wApMRH85fQ0zvj8OXJTptJj8rUN+iilEBG+2JpHSWVtHRHJGYNTuN03nR/UfGsYkkD4cyMcLPWwu6DCCMndOB/evd40PALvXMe0E15iQXk1+eXVpMQ0PfZwKKWo8ddQ5a2i2lfN/pJSbnxtBcU+N6tza3hvezlWWzWVtZVUeCuoqK2gsraSSm9l8DXQ5vF6+M8Z/yHd1b63yfYwBEcFEbkOuA6gT5/wURNdAkc0GcpCdvXhSCGLRbh1zhCueWE1zy/fQGlNKYOrKiF9AoiVadve4OOkOJ7fvowLx59wePpan6oysFjB0bg7pAG7F0NEHPQaF2zqlxzNqUNTeW1lFjeeOogIm3nzEIGBp8G6V43s0Uam6gHUnq9ZH2HcFD/I+ZoxLR8VAPsPrARgxe4oBiS7+NePxnPmI4uxKkVRU5Ww3HnkW60MNIvPNEaU6TbyeIoIFSRXSvHn9zfx3c5CHrpkbB2X2lTHHl52Gk+eby6xc84APwdLqjhUnIqK2s9EZ92n/7NMQ/C/jQe5cdagOtvKK43omF2FNipqfPzYDDYY2SuON38+lQ82HOTvq+0U+8socVjpZa6TNEWSM4ndVmvQEAD8cGJv3l93gE83H+L8cS27gTzz7R6Soh1cMD6dv+w2qpPVD9kMJdJuJSUqjjLAXV1MaKzWjtxy/vP1Li46IZ2TW5jHc1HJS/iliqvyf8hDeW4G9zCMulKKh9c+zJvb3uS8geexaetoesZFMyPkYSg93snBxCmUVsYTt/GtZg3B0p3G/+LZ1R/Bgjug74lw+evUVhbhfvY0Rm39P1yR83hnq4W+qUJZdRkl1SUUVxUHX4uriymtLqXGV4PX7zV+lJdaXy3ecLkqKRBt/ir+tLzuJotYiLJFEWWPqvOaEpVClC0Kawd4z9vDEOQAoZrOGWZbuD7ZImID4oDCFu4LgFLqaeBpgIkTJ3bpoN4Mm4u1vorgUw3ArKGpnNAnnv+u+A6SYXBtrRG7nzKUaWuegaQ4LFE7uHXO5eEP6quFZ06DmDSY14rInt1fQ/+TDAMSwrwT+/Hj51by0caDXDg+JClr0Gmw6r/GWkFApbQRcnd+Rr7NRiTCJzV5/M5Xi91q3ETfWZvNE1/t5EdT+nLl1D6HjU0Im/evA6DGMoQXr5lMWlwks4amsd0nFNU0MVV351JgtTC5iYghAKe5SOupKqljCJ5ZsofXV+7nhpkDubi+LpWZSBZvS2XZVi+3vr2eoooaVG06ubb1lOduJPS5PT3eyQmme6iBITBrEXy728/YjDjG9Y4PbhMRzhvbize3xFFUW06h1cKwRqJ2QkmMSqXQYkW5cwnctqcNSCI93sn8NdktMgQ788r5clsevzp9MJF2K25fFa4WeJH7JCSyyQflIYvFfr/i9nc2EhNp445zGl/fqEPeVmTNc1SNnUfO5r789MXVvHfjdBKjHTyx7gme3/Q8Y5LHsCBzAbWWN0jvN54VB51M6zUt+P80fUga76+eylU7PkU8JRDGiHr9XrLKsnhv+2LOSn2X17Zs4MCAkRxIcHHgvXMpri6G1GjAiyQ8y+NbgRBtwWh7NPER8SRGJpIUmcTAuIE4rA5sFht2ix2bxRb8ibBGgN/GK8sPklvi48aZwxnVK4nb5m+nX0Ii9184CZfDRZQtCqfN2aTrrCNoD0OwChgsIv0xbuKXAT+q12chMA9YBlwMfKmUUiKyEHhNRP4J9AIGAyvbYUzHNOmRSbhrsymrKSPOrJ0rItw6ZyjzFiwkEhjit0DyULBYiXf1o1eNF5WeTXp8+Jhz1rxgyCIX7IA930D/k5sfSNEeoy7xtJsabJoxKJkBydG8sHQfF47PQCnFkpwlvJ21kIujXZyy84tmDcG6g8sgCq5JnsSTBSv5NvN9Zg79AY8syuTRLzJJjYng7g+38Px3e/jtnKGcO6ZXMKqpwF3N1zvXQzw8cNklQTnvyyf34cFvreSpxiN9qsuyKbNaSWlm+uyMMA1BSFWtzzYf4r6Pt3L26DRund0wy1rt+47vo6I5ue9UevUayoOfbgfg6tMnsSDnI3YWbae+A+ycMb24+8Mt7CmoCLqYAMpNX/q+Yjt/n9sv7Bh7RCaS6T5o6AxFNZ4TESApJoMai1BRlh00bhaLcPGEDB77MpOcEk/j3yGTZ7/di8Nm4Soz6MHtq8bVwHnWkIHJyWzKhbIQQ/DayizW7CvmoUvGNhpSXQel4NM/QEQM0bP/xH8nWLj06eVc/8oaTp2yiac2PMUPBv+AP0/7M//8ci1Pf/8qFRlr+fminzMgbgA/HPpDMlwZ9Ojh4VXrIKZYviBq3UvYx/yQ/eX72VG8g+1F29levJ1dJbuoDqilJsEq4ukVk0ivyERGJI8iJSqFWEcssVmriF33Optiz2LOZXcQ44ghITIBh7UF12NSVevj6udXsn9fKk9fNYHThht/y7MHpfLKin2kRGY0EPQ7mrT5zKbP/ybgU8AKPKeU2iwidwGrlVILgWeBl0VkJ1CEYSww+70FbAG8wI1KqVbIEHZNMlwZUJxNdtk+4lIOO0xOHJhMWkoReAVX0jCw2lBK8Z5vOqd6FvJW5FaqfdXG00UoVWWw+H7ocyIU7zHe9zupyak8ABvfNl7D3NAtFuHH0/ry1w828vjKN/ji4JvsLNkJgDelF6fs+hK4u/Fjl+xnQ20JEZLA1WOu5Y3Pl7Fw21ssXD+Id7/P4ZIJGdx74WhW7Cnkbx9t45Y31vH0N7u5/azhjO8TzzUvrCLNUkyCH8ZmHM59mDk0hSe+cVBE44agsDQLgOS4pl2IyvRkL962hxVb17Ixp5SsokrGZsTxj0vGNQy19XnZc2gtRWkJTOgxgQsHDcTrU+wrrODaKakseAd2VuVxQq0H7IdvtmePTuPuD7fwUT33UJlpgGIj4jinkfoYKVE9+NK+HZ8ISa4wOSD1SIo1JtiFZfvrzHIunpDBo19ksmBNdh2dqfoUuqt5Z202PzghPejLd6taelmav1UMS0nh/VworjLCV3PLqnjg422cODCJC8f35K3tb3Fan9OCiW9hyfwcdn0Jc/4G0UmMj4YHLx7DrZ8+ztb1H3JO/3P409Q/IQgffe9mnOuHvHLJA3y691Ne3vIy96+8//Cx+sHF9IQd/zV+TBIiEhiSOIRLB11I3x1LGZazksq0C5jy4/8g1jAGb9iPWLpxPzfmvgt5l8LQs5r9XYTi9fm56bXvWb67iEcuHRc0ApTm8CPHNzjU9+TN/5T+UVVQkW8UObI6YNDpMGSO4RkQIa+sitdX7ufH0/rWWYdqD9rFBCmlPgI+qtf255D3VUDYen5KqXuBe9tjHF2FjIRBULyc/fmbGZlS13MeG1tAWl41Wy39GQV8tiWXxwsncF/8W7wSZ8hNTOs1re4Blz5mFJSZ8zZkr4aPfwt7lzQ9KyjZD0v+achMpwxpsLmythJfzBJcg/7LU1tLGBQ/iPtm3Mea3DV8vPN9anN3YC8/ZLiiwrHnGzZERDAybhDOXhM5s7KGtyzbKN6xk9/OGccNMwciIpw0OIXpv0xm4foDPPjpdq58dgXJrgiKK2uI719N33r1BGxWC4kOFwf95eQUV5Ke0HABM98sZRmo7VufVXuL+O3b6ymr2gd9YNmu/WyKKGF0ehyXTe7N5ZP61MkuDXJoPWtsRibyCaknICLccrpxU1VKEW2JINNuM2ZlPQ+Xx+wZ52RC3wQ+rOceKjTDLM8fPaDRRdyUmHR8hwyDlBzXeFhygESzjkFRxSFCe/dOjGLagCTmr8nmplmDGhVefGV5FtVePz+d0T/YVqG8xFgaXygOMKpXGmyCMjMc984PNlPt83PvhaP57sB33L38brLLs/m/if8X/gC+WiODOHFgnWCEaud3RKZ9SG3ZKAbwU6wWK2v2FbG7oILrZxrumHMHnsvcAXPZX76f8ppyKr2V/OWD75lY8RmTvEvxnHEnPZOGMTRxKCnOFOTAWljwM/xFe3mw9of8cO4/whsBABFWj/wzsct2MfKda5Frv4Lkxo1pff7w7kYWbc3lzvNGcsF4c5ZaWwXPzWFQ6X5us0PNzgiITYXoZGPxoKoEFv8NFt8HMT1hyByWVozmP+sTOX9cr2PTEGhaR0bqKNgN2QV1i5l4/V4OVe7j1JoqFhQnk+au5v6PtxGZ0p8J8WOxqYMsO7C0riEoOwhLH4dRFxsiXKkj4Nt/wld/a3pW8OkfjNc599Vp3l++n7d3vM07me9QWl1KcuRQDmZdwFMX3khqbCQR1ggWZC5gc4SDcbu+hHH1vYAGNbu/YmuEgx/1msre4mqGVCTjjangytNKGvjKLRbhgvHpnDU6jZeX7eOFpXu5/4Lh/Gejj6kRDSOi+sQms6U8l3eXb+ems8Y32F5gyk8kRzXcd9XeIuY9t5LUmAgunDCMt/LhhxOT+fdZp4b/PYWybxlrIyNJikgIFigKICIMiu3LzopSQ3MoxBCAET1014db2J3vDsp/HygvgSi4akrDUpoBUs2SlQDJTegmBQjKTFQ2jKr64aQMfv3mel5YupefTO/XwA9dVevj5eV7mTU0hUGphyOuyvETXX8WGoYBicYaRqW/mi+25vLRxkPcOnsI/ZOjuf/zVwH4dO+n/HrCr8P7wFc/ZxjRy98Am3GjW7hrIfcsv4eT0k+mVq7i/o93MDg1jk83HyLKYeWckCxrEaFP7OFZ4FkD4lmwyMJdEZ9DRRWMPcmoW/Ddo/Dl3eBK4+H0f/J+QR9+l9x0gMXgjFSuq/k130T9FdvLFxoPWa4exoOQK9Wo8x3fp24tD4xkt7dWZ3P9KQPranyt+i+U7odLX+W29Ul8sqOc1Tefjs0ashZTUWBEPe34BLVxPhfUvMDZkQ4ctV9Aq0MvmkZLTHQCUUlDSPT5yCnbW6c9qyyLGn8tg2trWV3dh0ufWsaeggr+cPYwosf+iPFVVSzb92Xdgy2+D/xeOM3MurRHwoz/g6ylxlpBOHZ9adQXOOk3EN8Hn9/HN9nfcMOiGzjnnXN4afNLTOoxiZfOeoln57xAddkw3lxlPGVPMqUeVsUmwc4vwh9fKbbt/5YaEeKtg7no30sp9AynX20t+6u/bvT3EmGz8rOTBvDt709lbj8PeTZrnX/sAGkxKbgtFr76fkNYMbCCKiMap768xOq9RVz93ErSYiN56+fTuHjySAD8qoV1i7OWsTYqmhPSJoa9kQ1KHkmmw4E61LA2QSC5LCA5UVXro7imAquCAUmNRzcFsosBkqKaj7hJjDRuxoXVDaOqzhrVkxmDkrnrwy1c88Iqcuvliby/LocCdw3XnjQg2KaUogJFjK35GYHL4UIUVKla/vTeJob0cHHdyQPZXbKbpQeWMiRhCAcqDrCpIEzthsoi+Oo+ozrekDMB+CLrC/703Z+Y0nMKD8/6J49eNpFhabH88vXv+WD9Ac4e3bNJv/pJQ1LYq3pSnDAaNrxl1Cl45UJY9BcYejYrzlzIE7tTmT0yrdnF2RG9YjlAMovHPWLc/Hd9ZRiUj38Hb18Nz58JD4+EdXU1jp77dg/RDis3zArJ4/GUwJJ/GBF4w+cyc1Q/SiprWbW33t8sOtl40PrhSyw4fQlX1NxOwbCrjDDudkYbgs4grjcZtV6yK+qGGu4oMXTfB9f6GDRqErvyK5g+KIlZQ1NhxAWcWF3LNncWBYHyhXlbjUpNk68zFEIDnPBjiOllrBXUlxbw1sBHv4OE/rgn/YRnNz7LOe+ew41f3Mi2om1cP/Z6Pv3Bpzw862HGp45nYIqLkwYn88qKfdT6/CREJjAkYQgr4pINgxJOz6Ugkw1+Q6LizW8tOKwWzjv3Is4tr2Bt/jpy3M1r32TlGDEDfZMaPi0nmoumNZ79fLOj4ZNvQU0ZglHkPcBqcybQIzaS16+bSmpsJE4zfLSyGYkJAJTiYPYKDlhgQo8JYbsMThxKqdVCQV7DG13POCcTTfcQwP82HKRGanE1UosgQCC7GJqWywgQuOZwUVWRdisvXTOZv547gmW7C5n98De8vy4HpVQwgWx4z9g6yrkerwefCK4WGAKLWIjAhseiyC91c9+Fo3HYLLy69VUcFgf/nPlPbBYbn+37rOHOX//dkKaYcx+I4Fd+/rH6HwyOH8yjsx4lwhpBlMPGM/MmEmm3UlHja7bSYKDAzDcRM+HQBnhiCmStgHMfI++sp7nxnT30T47mt3PCSK/Xo3dCFK4IG0uqB8LPFsFvtsKfCuDWnXD9d3DlAug9GT7+PZQZOUJ5ZVV8sOEAl0zsTWxoadzvHjWqrJ3+FwBOGpyCw2bh8y3hM/aVUjy7LIeClBPp+cN/BGdL7Yk2BJ2BPZJ0bGQHyhCaZBZnYgUGxA/kV2eOZvqgJP48d6Rxo3DGMy3VuAEtzzE1VBb9FRwxcPKtDY7PSYFZQb0n8OVPogoz+WTS5Zz34Q95ZO0jpLvS+ccp/+DTiz/lhnE30CO6bnTK1Sf2I7esOqiHNDltMuv8FdR4iuDguobXt+drNkQ4iLclsOOAhdvPHkbaiJM4p8K44f5v9/+a/RVlmU+NfdIaJqElxhrT77ToYl5fmdVge763gkRxBJVh1+yrawR6xBoRSIGqYJ6WFFwvyGSNGanUqCGIN/zGAa2o+pwzpifbDpWzK9/NS8v2Umv1EddM5EmqKevQnFxGALvFTpzYKfSFn+VYLMLV0/vz0c0nMSAlmlveWMcNr67l3e9zyMxzc+1J/esYJneNsfDrasG5wVB0dVuEqyckMLFfIqXVpXywayHn9JpB35yNnBg3hE93fYiqKDz8kJK/w3CVnPBjSBsFwKpDq9hfvp+rR11NVEgiW694Jy//dDK3nzWMyf2bDqe1WoTpg5J4qmg8yuaE+N7w82/wjf8xN7+xDnd1Lf++ckKLonUsFmFYWkzdDGOLBVwpxpgHnQ4X/Bt8NfDBr0ApXl6+D69f1ZXTKDsIy/8Noy8Jug+jI2zMGJTM51sPhdWEWr67iK0Hy7hmRkN3XnuhDUEnkeGI5ZCvCq//cLJJZnEmfbx+InqOo29SNK/+bCpD0w77aoeNu5p4n49lO96DPUtgxydw0q/DyvIy/qqGs4LSHPZ89xDXDRjObzNfIdmZzKtnv8qzc55ldr/ZDUT0AswcmkqfxKhgBbbJaZOpVl7WR0bAzi8b7rB7MeudUbjL+zI63VQ8dcaTnjCECTj5YNcHzYqg7TPVRfukNvSFJpruopE9a/hiW15dKQy/j0JVS7L5BLtmXzHznltFaj0jAOC0BwxBC2QIspayNjICly0qeMOvT0BzaGdtaVhN/rNG9UQE/vbRVtZnl1Jt9RPTjO/dZXfhtEaSFN28+yJAki2aIlFQ7W60z4AUF/OvP5HfnzmML7bm8X9vradHbARz66nTus1cB5ej6SztAAn2KNwWC7dGfwwLfsY7L52Gx1fFFctfhTevYM6OJRysKmDjY8Pgnh7wyGh46TywOWHWHcHjzN8xn1hHLGf0PaPBOYb3jOXnpwxs0e9jxqAUtpRFsveKJfCzLyFlCI8s2sHy3UXcc8FohvRo2XWB4R7aerAcf2O1CZIGwml/hsxPqVn7Gq+uyOL04T3omxRiRL++33DlzvpjnV1nj+jB/iIP2w41LILz3Hd7SIiytzgZ8EjQhqCTyHCm4hXIrTw8Hcws2sbgKk+DhcYA1sFzmFrjZ1nBetTnd0BsOky5PvwJgrOCZbB7MR6vh8c+/AkX9YhnsxX+MOUPvH7O64xJaX7RyWqGkq7aW8zmA6VMSJuARSysSukLu+qtE/h9FGR9ywGr4C5L5/azhh2OUOkzhXNLCtlbtpfNhZubPGdW5SESleCKaPiPmmj6zdNiPPj8qm6Jw4oC8q0Wkh1xvL8uh3nPrSQlJoLXr61rBCBkRuBrgSHYt4w1UdGM63ECVkv46JLEyEQS7S6zNsHWBtvT4iKZ2DeBRVvzSIhQuAViG6lFEEBESIlKJbkF6wMBkiLiDJmJ8iYksDH+rr+YOZCFv5zO9EFJ/HbOMBy2urcEd4VRh9vlaF7eAiA2Mha3xULEyifxZi3ndYePSY5khs59HK79kpkXvIhNLHw2cg5M+Tn0ngrJQ+Cch4yna6CoqohFWYs4b+B5DUOlW8lJgw132uIDNrA5WLw9j399uZMfTsxomCzYDMN7xuKu9pJd3MSaUuCaPv49torcOtFXFGTC2pdh4jXBrO8Apw3vgQgN3ENZhZUs2prLFVP6tlge5EjQhqCTyIgxok6yTTnlytpKsisOGhnFaY3cnG0OpqWMJV/Vsit/M5x6R5149QaYawXfL76TC+efxX+9BznT1Z+FF33I5cMub/SGFo5LJvTGabfy4tK9xDpiGZ44nBVRLti/su7T78H1rMdI0hmbMoYTQ3WQek/hjNJCHBY7H+xqOvs5q7acvtbw7ohEs9hMtb+EqQMSeWNVVvAprbrkAAVWK1l5Vm55Yx0ZCU5ev3ZqMCEtFLvFjl2BJ5BU1ATFB9ey22Zp1C0UYHDcQLNa2Zaw2wNRLpePjmuyFkEoZ/Y/k9P7nN5svwCJUakUWazGjacFDEuL5dWfTQ17YwzOCJqR6wgQ7eqJO20k3JbFVxc9zEG8XDH9Dhh7GaRPIHbQGUxPP4lPa3JRZ9wFP/gvzFtobDdZuHMhXr+Xi4dc3KJzNkXvxCj6J0ezJLOAAyUefv3mOoalxXDneaNafawRwdoETSi5Wqyo8x9Heat5LOZFpvQL+b19cZfx/3rybxvslhITwfje8Q0MwQtL92IV4appzYcOtwVtCDqJjCQjdj873yipF0jWGlzjDfpJw3HimGsAeLdHXxhzadMnsUWQPeWn3CR5SEU+z5XD385/q0WLjvWJi7Jzwfh03l93gOKKGianTWZDbbFR4Ss0OmnPN2yIcICy8Kcz6k3re08h1q+Y6erPJ3s/odbfiASvt5os8dGnEYkIl92FTRk1By6f3If9RR4+3nSIx7/M5DfPf0qh1YrDkshTV03go5tPCmsEAjix4PHXNHv9a70lQOPrAwEGp4xil92BPze8ITh/XDpnj07jxyfEU24RYlrgcvnl+F9y9airm+0XICmuL4VWqxGK2VJ2fGZkmtcjYAiinS0zBDGOGNzKC5FxvLLlFdJd6czMmFmnz5x+czhUcYgNBQ0VaZVSzM+cbwQqxDeimNtKZgxKZvnuQm56bS01Xj9PXHFC+DyRZhiaFoNFaLaG8bfF8fy99hKm1q5ENs03GrNXG5F6J/4yOPOpz+yRaWzMKeVAiTHjKK+q5a3V+zlnTM8Gs9n2RhuCTqJHykhsSpFTZDy1BaqSDYnqBWHcIQHSBs3moqh+vGSvYeGephddPV4PvypahhILTx08xKQz/t70DKIZ5p3Yl2qvn3v+t5UTUifhVT6+d8XXCSOt2vElKyJiibP2Z2xGvS984gCISubcWitFVUUszQlfgrMyfyv5NmuDWP0AIkIiVopr3cwZmUZ8lJ0bX1vLQ5/tYGhCOV4RLp44njkj0xpNnArgFAseXzOGwO9njdTiwMLIpJFNdh0UPxiPRcjJC18zNyHawZNXTCDNUUO5xUJsRFzYfm0hMTqNcquFmvztLdvBVwtvXgmvX2aICYZQYYr7xTSVDRyCy+GivKacrYVbWZu3NuzMc2bvmdgtdj7b2zB6aHXuavaV7WuX2UCAkwYnU1njY21WCff/YAwDU1zN7xSGSLuVASmuZovUPPvtHj6IPB9/xiT46LdG/Y5Ff4WoZJh2Y6P7nTHCmOku2mrMCuavycZd7eUn0/s3uk97oQ1BJ2FN6E9Pr5fsMsM1lFmSiVNBeo/w6wNBRLjjoneYnDaZvyz9C6sOrQrbTSnFncvuZEdxJvePvoE+p/yh1anx9RmWFssNM426sv/4oAarWFmVOsBYJ1AKvNX49y9nR4SFWf0mhh07faYy/dAOEiIS+GB3ePdQ1gFDo71PUuMiZUkWB0U+D5F2K7efNYxLJmTw0c0nceYIMws3JBGrKZxixRNOHTKU6jLWRTgY7ezRrL5McMG4dFfD0N0QgrUIIuJbNM7WEEwqK2ykEHx9inaDrxrytxmFhEIoNyu9RbdwFumyu3DXuHll6ys4bU4uHHxhgz4xjhim95rOZ/s+w6/q1ox+e8fbxDhimN13dsvG3gKmDUzCFWHjx9P6cu7Y5mU6mmJEz9gmi9TszHOzeHs+V544AMsF/4Zaj7EYvncJnPK7Jh/yBqa4GJASzedbcvH5FS8s3csJfeLriBF2FNoQdBZxGWR4fWR7jMW4zIItDKquxtKrGUMA2K12/jnzn/SO6c2vvvoVe0v3Nujz6tZX+d/u/3HjuBs5eeINRvJYO/C7M4fx7ytOYG+eF5+nN18LUJIFhTvZvW4x++x+ai2KaY3VHug9GXvRHs7MOIWvsr4iu7xhpbF9BcZCcp+0hlnDARJsURSbrqVLJ/XhwUvGMqJXLPlmnYeUmJZFWDjFhqc5eStPMbk2K32djVfoCjAo3jAEmarakIJuhEAtgpY+abeGwExqd9m+Jo1RkMB6RvpE+PYROPB9cFOFKSDnim7ZYrXL7qLGX8PHez7m/IHnE9vIIvPsfrMN91D+YfdQSVUJi/Yt4twB5xJpaz9XSEykne9+fyp3ntf0bK4ljOgVS06Jh6e/2UWN199g+/Pf7cFhsxi1DZIHw6l/NAxsfF+Y8JNmjz97RBrLdhXy/roc9hVWcs2Mjp8NgDYEnYfVTrpEkFNbhlKKzOIdh6WnW0BcRBxPnPYENouNG7+4keIQff5Vh1bx0OqHmNV7FteOaVjUo62cNbonH/xyBrEMI9NbgFsEX+Yi1n+zkHVmLd+xqY1cR++pAFwW1Z8IWwSX/+9ylh1YVqdLVmkgdHR0o2NIdMQYIZL1SmAGpBVaug7itNippGlDoCoLKbFYiWvB03u0PZr0iCQzcqjxyKgyc5wxYWQw2sqQBGP9abvUNGmMguRtAwQue9WQS3jvRiPxECivKcfp92ONjG/RuV0Ow+1S66/lR8PDy4/AYffQp3s/DbYt3LWQWn8tPxjygxadqzXERdnbJQb/skm9mTU0hfs+2sbsh7/ms82HY/9LKmtYsDabC8elkxwovjPtJph6o5Fj0IJEsDNG9MDrV/z5/c30jItkzshGtLzaGW0IOpEMRzxFysv+8v0UeysYXNNExFAYesf05tFZj3Ko4hC/+upX1PhqOFRxiFu/vpXeMb25b8Z9WKRj/sT9kqO5/+yLQBQfRaSx+ov5ZJSsZLUrlaTIJHpFNzIF7zkWrA4GFOzmjXPeINmZzPWLrueFTS8E/6GyKnNJVk0nUCVEJFBktRhieyHkmwaxvrxEYzitDjzNPDVXVeRRYxHiWmhcBiUOMSOHGoaQBgjUIoipl7zXHsRFxNEzIoHtDkfLFozzthjhjDFpMPcRw4AteQiACm8FMX4/OFrmV3fZjX4z0mfQP67xp9kYRwzT06fz+b7P8St/cJF4TMqYoCE7FomPcvD8Tybzwk8mYbNauO7lNVzxzAq2HizjtZVZVNX6+cmMfod3sFjhzPug3/QWHX9873iSXRG4q738eFo/o3TmUUAbgk4kI9oIJfxq/1cADHYkhE8Oa4JxqeO4d8a9rM1byx3f3cGvv/o11b5qHj310eDTWUcxqed47BY7G3r2Y0ztBsZbdrEtOpIxKU3UpLVHQq/xsH8lfWL78OrZr3J6n9P5x5p/8LtvfkdlbSVZXjd9bE2PPTEqBY/FQmVp3Vq8BbXlOLHUyUZtiihrBB4hvFSGSanbyKiOa2Es/6CkEey126nNDaOpY1JuGqzYI4jgaglDE4aw3WFvmSHI32aIFQIMPRPGXGZo4RzcQHltJdF+1eKqd4FM6KtGXNVs39l9Z5NbmcuG/A2szVvLntI9XDy4/RaJO5KZQ1P5+JaTuOv8kWw5WMY5jy3h8S93MmNQMsPSWpZzEQ6LRZgzsgdRDiuXT+7d/A7thDYEnUiGqSb5VZaRnTu4icXRpjiz/5ncPP5mPt7zMZsKN3HvjHsZEDeg+R3bSKQtknGp49gRBU6pwW1RZPnczSep9Z5s+KFrq4iyR/HQKQ/x6wm/5rN9n3Hlh5ex0+KnbzNP9EnRxpS5uN76SIGvihRry/3LTmskHosEi72HI2gIXC2bpg9KGIRXJLjWEY5ysxZBTMSR3zSaYkjKWPba7VTnNz4rAYwoocJddYXMzvwbRCXhfv8XbKsuJF5J87UtTKb0nMJ757/Hib1ObLbvrN6zcFgcfLr3U+bvmI/L7mJOvzktOs+xgN1q4cfT+vH1rbO4+sT+KAU3zGx7yOvtZw/n41tOIj6q/TWFGkMbgk4kI8n45/s+73uSvD4Se4WJtGkhPxv9M34x9hf8ccofOa3Pae01xGaZlDaJbZUHKLU52OA0nsLHpjSzztF7qqHJcnA9YISDXjPqGv496iZyS3ZTZrXSp0fjC8UACTFG8lNR6IygpoIC8TdbtD4Upy0SjwjUNq43VOox/PlxLSgMAyGaQ2VZhuxxGMpMDZ+W5BEcCUMTh+ITYWcTxggwks6UD1JDxP2iEqk96+/8ijwO+iq5vrrlvnWLWFoc/+9yuJiePp2P93zMZ3s/45wB57R4JncsERdl58/njmDLXXPqJlAeIa4IW11ZiqOANgSdSGzSEGJ8fvwoBtfWtHihOBwiwg3jbuCyYZc137kdmZI2BYVidf/JrE8bikWaj7Wn92Tjdb9Ztdvvg68f5MQPfscbbjvn9TyJOWbiXGMkxhuRMcWhCq5m0frkiJYlPwE4bVGGIWhqRuAxSkrGtaBmMED/uP5YETKtfigIH8tfXmvoAHWkIQDYUd5QlK8OgXWMEEPgV37+VLScFc5I7iwo4kTpOBfj7H6zKawqpMZfwyVDwtau6jIc7TrD7YkuTNOJSEI/MrxetlodxkJxz/YtNnE0GJ08GqfNycqBJ7K7dA9DapKaf6pzpRrJZftXGsk271xrqKSOvoTecx/m3iZirQMkxpglGStDomLceUZt3xYuFAM47VFUWyz4atyNVuUtNWPp41qY/OWwOujrymBnRYUhDhj6tG1S7vVgs0JkK9xYraF3TG+cYmO7r9gQn4to5GaevxUsNkg6LKT36NpH+d/u/3HLqGs578vHoIVZxUfCzIyZOCwOhiYODRovzdFHG4LOJLYX6V4fWyNgsDgbL/t4DGO32hmfOp7lh1aSV5nH2f3PbtmOvafAto/gPzOguhzO+5ehmNrCp6pEM/6+yCwAD1BVlk251UKKK3z933BEmW6kKk8JjU3GS2sNN05LDQHAoKThbC3dbxi4Kdc12F7uqyLWaus4WWGxMCSqF9sq3VCYaSzQhyNvq1Ea0gxtfG3razy36TkuHXopPz3hl9B7Nqjw7q32wOVw8dApD9GrhW43TcegXUOdicVKhqk+OSRhUDOdj10mpU1id+lu3LUtWCgO0HsKVJcaUVLXfmkI5LXipui0OYlUUFx9OMuzoMTQykkKU9Ws0eOYkVWeMBW9ApTWVuBAWvX0PihhENlWqNz3bdh1gnJ/DTGNyH63F0MTh7LD4UDlNxE5lLcVUo21qkX7FnH/yvuZ1XsWt0++3TBSqcOgx5EFMbSUWX1m6dlAJ6MNQSczOTKVQTU1DEw78oXizmZK2pTg+2YXioMdLzeSbK798ohuNCJCgtgo8h7W3C8ws5RT4luejek0ffSeMPUDApT6PMRL657exySPQQHrqILchrpDZcrXbC2CtjI0bSLlVgsHc9eF71BTCcV7IXUEa3PX8vtvfs+YlDE8cPIDrVKm1XR92mQIRCRRRD4XkUzzNawzUUTmmX0yRWReSPtiEdkuIuvMn+Zz+I8zToofzrs5h4js1YgkQxdgeNJwou3RxDpiGxWKa4A90qjH2sL49HAkWiIpCpGQLqgwwjyTo1v+NXKa4ZuVYUo7Bij11xJrad1Ne3zqeGxiZWVkhLFOEIq3mnJRxDRTi6CtDEk2DOz2xiKHCrYDCl/yEG79+lZ6uXrx+KmPB+s0aLoPbZ0R3AZ8oZQaDHxhfq6DiCQCfwGmAJOBv9QzGFcopcaZPy3Ihz/OCNQabkPEUGdjs9g4f+D5zB0w96hGTiTaoynCG9TTKTCzdVsjs+00/f6e6kYMgd9PKT7iWnlzjLJHMTJ5FKtc8XVlugGqygzlUXvHJvwNSRiCANvdDfWcgGDE0CaHg3xPPjeOu5H4FkpJaI4v2rpYfD4w03z/IrAY+H29PnOAz5VSRQAi8jlwJvB6G899fHDCPEOQKqFjC090NLdPuf2onzPBEccOS46x2BwZS35NKRabIT/RUpyRhiGorGlEY766lBKL0MfW+pnL5LTJPJe/gYqspUT7vGA1/92qSo2iNB0UOhogyh5FH6uLHd58Q5PJWu/fPW8rWB18696DRSxM6zWtQ8ejOXZp64ygh1IqEMh9CAgnnJIOhOoAZJttAZ433UJ/kiYeJ0XkOhFZLSKr8/Pz2zjsYwhXCozp2vHTnUWSM5FiixVlCqsVeitIEker/NtOs/KWp7E8Ak8xZRYL8UdQN2BS2iR8KNZaauHgusMbqkqNojQdUIugPkNc6Wxz2KBkX8ONeVsheQjfHljK6OTRrYqK0hxfNGsIRGSRiGwK83N+aD9lKIa1QPO2DlcopUYDJ5k/jQqUKKWeVkpNVEpNTElpeZy45vglMSqVGotQUbYf/H7y/TXBovUtJbBG4PE2bghKLZYWKY/WZ1zqOGxiY1VkpBFGalJdmU+1xUJsC8s/toWhSSPYb7dTkduwGhj52yhKHsjmws3MSJ/R4WPRHLs0awiUUqcrpUaF+XkfyBWRngDmazgffw4Qqp6UYbahlAq8lgOvYawhaDQtIsEU7Ssq3QeeYgqsQlILi6wHCBawb0Riosqda9y0j6BugNPmZEzKGFbGxNdZJyg3C8LHRLZOYPBIGNrLiOjKPLCy7oaqMijdz9JoFwrFSekndfhYNMcubXUNLQQCUUDzgPfD9PkUmC0iCeYi8WzgUxGxiUgygIjYgblA43KNGk09EmON54ui8mxw51JgtZIS2bobdtAQeD1ht5eUG57PliqP1mdS2iS2Wv2U718ZLANZHqxF0PEz26E9jGi07fWrlZllLL/1l5MYmcjwpIbZz5ruQ1sNwf3AGSKSCZxufkZEJorIMwDmIvHdwCrz5y6zLQLDIGwA1mHMEv7bxvFouhGJZjnKIvch/OUHDXmJVur7R5muJI+vKuz2UrOSWHwrspVDmZw2GT+w1uY3CpgTWoug46Ol06LTiFUWtlfk1N2QtwU/sLR8Nyf2OrHD6lZougZtihpSShUCDaQulVKrgZ+FfH4OeK5enwpgQlvOr+neJJra90WefIpL9uETITmmdRrudqsdmwJPSD5CKGWtVB6tz9jUsTgsDlY6nZyy5xvoNz1Yi6AjqpPVR0QYao9je2W+EWYbiMfI38aWqBiKa8r0+oBGZxZrui4J5mJrcVUJBWVGVExyXOvDcJ0IHl9N2G0l5tN7S5VH6xNhjWBs6lhWxcQbBcw5XIugsXq+7c3QmD5k2iz4zLoKAORtYUliLwRpUe0AzfGNNgSaLkukLZIoJRTVllMQKFof2/qqTk6x4vHXht1Wat602xJaOSltEtssPkpzVkFNZYfXIqjPkOQReCwW9md9d7gxbxvfRtoYlTwqaFA13RdtCDRdmkSLnUJvBQXBovWtX4CNasoQHIHyaH0mp01GAWscVti/grIOrkVQn6HpxhP/9kOrjIbKIkoq89jkczM9vWW1dDXHN9oQaLo0iVYnxf4a8qsNF07SkYR5ig2PCl+z+EiUR+szOnk0EVYHq5xO2PMN5V4Pdgy30dFgYPpUrEqxvcgskpO/jWXOSPwovT6gAbQh0HRxEu0uivBTWF1GdCuK1ofitNjx0Igh8HmIa6XyaH0cVgfjUscH8wnKfVXE0HG1COoTYYukv7Kxo9IUAcjbwrfOSOLsMYxKGnVUxqA5ttGGQNOlSYxIoNhqId/rblXR+lCcFjuVjRRfKfXXEmdpexHxyWmT2SFeig+tp9znIbaDaxHUZ6gjge0+I3van7uV76KiODF9upab1gDaEGi6OAnOJIqtVvKtFpJsR6bm6bRG4BHVsIBMUHm07QXVJ6cZSfOrI2yUKx8xHVSisjGGxvThkFUoLcthW/4GCq0WZmTobGKNgTYEmi5NYnQPvCLssdtJOcLoF6c1Eo9YoL7MRHUppRYh7ggNTCgjk0fitEayMiraUB49ypr/Q83KcduzFvNthRFqq8NGNQG0IdB0aRJdhpBtcSuL1ofitDnxWKShIQgKzrU93t9usTO+xwmsdsVRZrEQcwSy1m1hSIYRHbQ96xu+syuGRyS3qm6D5vhGGwJNlyYx7nDeQPIRykA4bU48IlBfirryyJVHwzEpbRI7pZZDNutRCx0NkNzzBJK9PlbnrmF9RAQzUrtuRTxN+6MNgaZLkxiTEXyffATJZABOexRVFgv+Gned9oDyaFw7PTkH1gmqLBZio45yVVabg6HY+Eqq8Ilw0oCzju75Ncc02hBoujSJIXkDyXEtL1ofitNuuGmqPCV12kvdpvJodPuohI5IGkG0ea6Y5KHtcszWMMSRhBIhxq8Y3fuUo35+zbGLNgSaLk1oWcojdQ1FmW6ayuriOu0lFYY2T1x02hGOri42i40TTJdMbDusO7SWobGGoZxGFDbr0Q1f1RzbaEOg6dLYrXZizK/xkS5+Ok1D4KkqrdNe5ikAIO4IDUw4JqVNAo6evEQoo3ucgCjFrNgBR/3cmmMbbQg0XZ6k6J5YxXLE4mnBcpU1ZXXaSz1FAMQfgWxFY5yccTI2i40+sX3a7ZgtpU/GVD7IPsg56bOO+rk1xzZtqkeg0RwLJESnUumvPeLiKkFDUF3PEFSXAG0TnKvPwPiBLP/R8qOmM1SHXuPpO/MOGHPJ0T+35phGGwJNl2dE0gji2xDi6TRnEp56UUMl5gyhvesGdIoRALBYYcavO+fcmmMabQg0XZ7bJt/Wpv2dkfEAVJp1AgKUeiuxWw/XNdZojlf0GoGm2+M0XT8eb93M4jKfh3ixHzWVUI2ms9CGQNPtCUhXe+pJTJT6a9pFeVSjOdbRhkDT7Qm4furMCPx+SpSP2HZQHtVojnXaZAhEJFFEPheRTPM1bPyeiHwiIiUi8mG99v4iskJEdorImyKiH780R53DhqD6cGNVCaVWC3FHWRxOo+kM2jojuA34Qik1GPjC/ByOB4GrwrQ/ADyslBoEFAM/beN4NJpWY7fYsSrw+KoON5rKo/HtGDqq0RyrtNUQnA+8aL5/EbggXCel1BdAnZAMMVbgTgXmN7e/RtORiAhOBI+/5nCjp4SydlQe1WiOZdoaPtpDKWUWQuUQ0KMV+yYBJUopr/k5G0hvrLOIXAdcB9CnT8OszNraWrKzs6mqqmqwTdN9iYyMJCMjA7u9aW2dKLHUMQRV7kNUWSzEtWNWsUZzrNKsIRCRRUA41a0/hn5QSikRUe01sPoopZ4GngaYOHFig/NkZ2cTExNDv379dLifBgClFIWFhWRnZ9O/f9PKpE6sePze4OeA8mhsVPsoj2o0xzLNGgKl1OmNbRORXBHpqZQ6KCI9gbxWnLsQiBcRmzkryAByWrF/HaqqqrQR0NRBREhKSiI/P7/Zvk6LDY86HDVUWpELQHw7Cs5pNMcqbV0jWAjMM9/PA95v6Y5KKQV8BVx8JPuHQxsBTX1a+p1wih2P8gU/l3aA8qhGc6zSVkNwP3CGiGQCp5ufEZGJIvJMoJOILAHeBk4TkWwRmWNu+j3wfyKyE2PN4Nk2jkejOSKcVgceDnscSz2FAMQ5EztrSBrNUaNNhkApVaiUOk0pNVgpdbpSqshsX62U+llIv5OUUilKKadSKkMp9anZvlspNVkpNUgpdYlSqrqxc3UFrFYr48aNC/7s3buXxYsXExcXF2w7/fTTufHGGxk3bhwjRozA6XQGt82fP7/O8d577z22bNnSbuN7/PHHGTRoECJCQUFBsF0pxc0338ygQYMYM2YMa9euBWD79u1MmDCBMWPGsGzZMgC8Xi+nn346lZWVYc/RVXFaHFSiQBnGoLTaqE0Q59Dho5rjHy061444nU7WrVtXp23v3r2cdNJJfPjhhw367927l7lz5zbYJ8B7773H3LlzGTFiRLuMb/r06cydO5eZM2fWaf/444/JzMwkMzOTFStW8Itf/IIVK1bw1FNP8eijj9KvXz9uueUWFixYwL///W+uvPJKoqKOr4xbpy3CKGDvrQK7k1JTebQ9Jag1mmOV49IQ3PnBZrYcKGu+YysY0SuWv5w7sl2P2RRLly5l4cKFfP3119xzzz0sWLCA8vJyrr/+eiorKxk4cCDPPfccCQkJzJw5k7Fjx/L111/j9Xp57rnnmDx5coNjjh8/Puy53n//fX784x8jIkydOpWSkhIOHjyI3W6nsrKSyspK7HY7JSUlfPDBB3zyyScdfflHHafVicciUFNpGAJvhVYe1XQbtNZQO+LxeIJungsvvDDYvmTJkmD7vffe26JjnXjiiZx33nk8+OCDrFu3joEDB/LjH/+YBx54gA0bNjB69GjuvPPOYP/KykrWrVvHk08+yTXXXNOqcefk5NC7d+/g54yMDHJycrjxxhu57777mDdvHn/4wx+4++67+cMf/oDFcvx9bZx2pzEjqK0AoNRXRZxWHtV0E47LGcHRfHIPJZxrCGjUNdQaSktLKSkp4ZRTTgFg3rx5XHLJ4UpTl19+OQAnn3wyZWVllJSUEB8f36Zz9unTh8WLFwOwc+dOsrOzGT58OFdddRU1NTXcfffdDBkypE3nOFZw2qLwWCz4q91YCCiPHl/uL42mMY6/R7tuSv0nVxFhzpw5jBs3jp/97GeN7GWQnp7O/v37g5+zs7NJT6+b5P3HP/6Re+65h8cee4yf/exn/P3vf68zI+nqOO2GuFxVdQn4fZQqH3FaeVTTTdCG4BgmJiaG8nJDoikuLo6EhASWLFkCwMsvvxycHQC8+eabAHz77bfExcURFxfHp59+yrp163jmmWcaHjyE8847j5deegmlFMuXLycuLo6ePQ/Hz3/99df06tWLwYMHU1lZicViwWKxHFeRQ06HCwCPpwSqSim1WIizuzp3UBrNUeK4dA0dL1x22WVce+21PPbYY8yfP58XX3wxuFg8YMAAnn/++WDfyMhIxo8fT21tLc8991zY4z322GP8/e9/59ChQ4wZM4azzz6bZ555hrPPPpuPPvqIQYMGERUVVee4SinuueeeoKG57rrruOKKK/B6vfz73//u2F/AUSTKEShgXwKeYkqsFkY4Yjp3UBrNUUKU6jB5oA5j4sSJavXq1XXatm7dyvDhwztpRJ3LzJkzeeihh5g4cWJnD+WYpCXfjU/XP8et6x7mnRE3MLjnJCZ9Po9L02dx6+zHj9IoNZqOR0TWKKUa3Ci0a0ijAZwR5oygppwqdy5VFgvxzuROHpVGc3TQrqHjgEBkj+bIcUbGA+CpLqes4hCglUc13Qc9I9BogKgIo8qqp9ZNidswBHGucOrrGs3xh54RaDQcnhFU1rgp9RvrZlp5VNNd0IZAowGcdiNnwFNbSZlZxD5eK49qugnaEGg0HNYU8vg84DMqlWnlUU13Qa8RtCPZ2dmcf/75DB48mIEDB3LLLbdQU1MTtu+BAwe4+OKLw24L5eyzz6akpOSIxvPXv/6Vhx56KGx7eno648aNY9SoUSxcuPCIjt/e3HfffZ12bqfdNATeKkprzCQ+rTyq6SZoQ9BOKKW46KKLuOCCC8jMzGTHjh243W7++Mc/Nujr9Xrp1atXg/oD4fjoo4/arBkUjl//+tesW7eOt99+m2uuuQa/39+i/Xw+X/OdjpAjMQTtNR6HxYFFgcdXbSiPIlp5VNNtOD5dQx/fBoc2tu8x00bDWfc3uvnLL78kMjKSn/zkJ4BRpObhhx+mf//+3Hnnnbz11lu88847uN1ufD4fL774InPnzmXTpk1UVlZy9dVXs2nTJoYOHcqBAwd44oknmDhxIv369WP16tW43W7OOussZsyYwdKlS0lPT+f999/H6XTy3//+l6effpqamhoGDRrEyy+/3OJ6AcOHD8dms1FQUMB1113H/v37qaqq4pZbbuG6664DwOVy8fOf/5xFixbxxBNP8OWXX/LBBx/g8Xg48cQTeeqppxARZs6cyfjx41myZAkVFRW89NJL/O1vf2Pjxo1ceuml3HPPPQC88sorPPbYY9TU1DBlyhSefPJJ/vjHPwbVW0eOHMmrr74atp/Vam0wng8//JCFCxdis9mYPXt22FlQc4gITgSPrxqPt4o4m00rj2q6DXpG0E5s3ryZCRMm1GmLjY2lT58+7Ny5E4C1a9cyf/58vv766zr9nnzySRISEtiyZQt33303a9asCXuOzMxMbrzxRjZv3kx8fDwLFiwA4KKLLmLVqlWsX7+e4cOH8+yzLa/4uWLFCiwWCykpKTz33HOsWbOG1atX89hjj1FYaJRrrKioYMqUKaxfv54ZM2Zw0003sWrVKjZt2oTH46mjrOpwOFi9ejXXX389559/Pk888QSbNm3ihRdeoLCwkK1bt/Lmm2/y3XffsW7dOqxWK6+++ir3339/UL311VdfbbRf/fEMHz6cd999l82bN7NhwwbuuOOOFl97fZxiweOvMZVHHUd8HI2mq3F8zgiaeHLvTM444wwSExtGonz77bfccsstAIwaNYoxY8aE3b9///6MGzcOgAkTJrB3714ANm3axB133EFJSQlut5s5c+aE3T+Uhx9+mFdeeYWYmBjefPNNRITHHnuMd999F4D9+/eTmZlJUlISVquVH/zgB8F9v/rqK/7+979TWVlJUVERI0eO5NxzzwUMATuA0aNHM3LkyKB43YABA9i/fz/ffvsta9asYdKkSYBRwyE1NbXB+L744otG+4WOJy4ujsjISH76058yd+5c5s6d2+y1N0YUVir9tZSilUc13Yvj0xB0AiNGjGjg8y8rKyMrK4tBgwaxdu1aoqOj23SOiIiI4Hur1YrH4wHg6quv5r333mPs2LG88MILLco0/vWvf82tt94a/Lx48WIWLVrEsmXLiIqKYubMmVRVVQGGoJ3VagWgqqqKG264gdWrV9O7d2/++te/BvuFjtFisdQZr8Viwev1opRi3rx5/O1vf2tyfE31Cx2PzWZj5cqVfPHFF8yfP5/HH3+cL7/8stnrD4fTYsXjraDUYiFdK49quhHaNdROnHbaaVRWVvLSSy8BxiLmb37zG66++upm/fXTp0/nrbfeAmDLli1s3Ni69Y3y8nJ69uxJbW1t0H3SWkpLS0lISCAqKopt27axfPnysP0CN/3k5GTcbneLFrxDOe2005g/fz55eXkAFBUVsW/fPgDsdju1tbXN9gvF7XZTWlrK2WefzcMPP8z69etbNZ5QnGLHo3yUWi3EaeVRTTeiTYZARBJF5HMRyTRfExrp94mIlIjIh/XaXxCRPSKyzvwZ15bxdCYiwrvvvsvbb7/N4MGDGTJkCJGRkS2KhLnhhhvIz89nxIgR3HHHHYwcOZK4uJaHLt59991MmTKF6dOnM2zYsCMa/5lnnonX62X48OHcdtttTJ06NWy/+Ph4rr32WkaNGsWcOXOCrpuWMmLECO655x5mz57NmDFjOOOMMzh48CBgSFyPGTOGK664osl+oZSXlzN37lzGjBnDjBkz+Oc//9n6izdxWhx4RIxaBBFhv8oazXFJm2SoReTvQJFS6n4RuQ1IUEr9Pky/04Ao4OdKqbkh7S8AHyqlWvVYebzJUPt8Pmpra4mMjGTXrl2cfvrpbN++HYdDL1i2By39btzy5hx2lu8jy27n5gEXce1Jx08FNo0GGpehbusawfnATPP9i8BioIEhUEp9ISIz67drDCorK5k1axa1tbUopXjyySe1EegEnNYIcs21h7iohgvYGs3xSlsNQQ+lVGC+fgjocQTHuFdE/gx8AdymlKoO10lErgOuA6Oo+vFETEwM9Wc4mqOP0+ak2mJ4S7XyqKY70ewagYgsEpFNYX7OD+2nDB9Ta/1MtwPDgElAImFmEyHHf1opNVEpNTElRevEa9qf0ExirTyq6U40OyNQSp3e2DYRyRWRnkqpgyLSE8hrzclDZhPVIvI8cGtT/TWajsQZkjsQF6kXizXdh7aGjy4E5pnv5wHvt2Zn03ggRi7/BcCmNo5HozlinI7DeR7xEfGdNxCN5ijTVkNwP3CGiGQCp5ufEZGJIvJMoJOILAHeBk4TkWwRCaS+vioiG4GNQDJwTxvHo9EcMU77YUOglUc13Yk2GQKlVKFS6jSl1GCl1OlKqSKzfbVS6mch/U5SSqUopZxKqQyl1Kdm+6lKqdFKqVFKqSuVUu62XU7ncu+99zJy5EjGjBnDuHHjWLFiRZuP+fnnnzNhwgRGjx7NhAkTGs2afeSRR6isrGzz+QJcc801pKamMmrUqDrtRUVFnHHGGQwePJgzzjiD4uJiABYsWMDIkSM56aSTghpFu3bt4tJLL223MXU0UWYSmQ208qimW6Ezi9uJZcuW8eGHH7J27Vo2bNjAokWL6N27d5uPm5yczAcffMDGjRt58cUXueqqq8L2a29DcPXVV/PJJ580aL///vs57bTTyMzM5LTTTuP++w1dp3/961+sWrWKn//857z22msA3HHHHUHF0a6AMyIWgDixa+VRTbfiuNQaemDlA2wr2tauxxyWOIzfT240qImDBw+SnJwc1NdJTk4GDHnqxx57jPfeew8wnvCffPJJ3n33XVwuF7/4xS/46KOP6NmzJ/fddx+/+93vyMrK4pFHHuG8885j/PjxwXOMHDkSj8dDdXV1HR2fxx57jAMHDjBr1iySk5P56quveP3117nvvvtQSnHOOefwwAMPAIak9LXXXstnn31GWloab7zxBuGisE4++eSgqF0o77//flDLaN68ecycOZMHHngAi8VCdXU1lZWV2O12lixZQlpaGoMHD27V77kziTLdQfGWiGZ6ajTHF3pG0E7Mnj2b/fv3M2TIEG644Yag1PSsWbPYtm0b+fn5ADz//PNcc801gCGnfOqpp7J582ZiYmK44447+Pzzz3n33Xf585//3OAcCxYs4IQTTqhjBABuvvlmevXqxVdffcVXX33FgQMH+P3vf8+XX37JunXrWLVqVdAQVVRUMHHiRDZv3swpp5zCnXe2Lns2Nzc3qCialpZGbm4uALfffjunn346H3zwAZdffjl33303f/rTn1p17M7GaRqCOO0W0nQzjssZQVNP7h2Fy+VizZo1LFmyhK+++opLL72U+++/n6uvvpqrrrqKV155hZ/85CcsW7YsKEzncDg488wzAUO2OSIiArvdzujRoxs8jW/evJnf//73fPbZZ82OZdWqVcycOTP4pH/FFVfwzTffcMEFF2CxWIJ++yuvvJKLLrroiK9ZRIIulDPOOIMzzjgDgJdeeomzzz6bHTt28NBDD5GQkMCjjz7a4mI5nYUz2sgmjo3SeSqa7oWeEbQjVquVmTNncuedd/L4448HC8f85Cc/4ZVXXuH111/nkksuwWYz7K/dftgXHSrbHJBsDpCdnc2FF17ISy+9xMCBA9t1zCLC/v37GTduHOPGjeM///lPk/179OgRFH87ePBgg1oClZWVvPDCC9x444385S9/4cUXX2TGjBlHrIp6NAmuESR2HXeWRtMeaEPQTmzfvp3MzMzg53Xr1tG3b18AevXqRa9evbjnnnuCpSxbSklJCeeccw73338/06dPb7RfTEwM5eVG0fXJkyfz9ddfU1BQgM/n4/XXX+eUU04BwO/3B6WjX3vtNWbMmEHv3r1Zt24d69at4/rrr29yPOeddx4vvvgiAC+++CLnn18nwZwHH3yQm2++GbvdjsfjQUSwWCztupDdUQQihXQOgaa7oQ1BO+F2u5k3bx4jRoxgzJgxbNmyhb/+9a/B7VdccQW9e/dutULq448/zs6dO7nrrruCT+0Bjf5QrrvuOs4880xmzZpFz549uf/++5k1axZjx45lwoQJwRt2dHQ0K1euZNSoUXz55Zdh1yIALr/8cqZNm8b27dvJyMgIlr+87bbb+Pzzzxk8eDCLFi3itttuC+5z4MABVq5cyQUXXADAL3/5SyZNmsR//vMffvSjH7XqujuDgCGINWcGGk13oU0y1J1FV5Shvummmxg/fjw//elPO3UcLpcLt7tLp2u0mpZ+N5RSPL3hac7ufza9Y9se+qvRHGt0lAy1pgVMmDCB6Oho/vGPf3T2UDRNICL8fOzPO3sYGs1RRxuCo8CaNWs6ewhButtsQKPRNM9xtUbQFd1cmo5Ffyc0muY5bgxBZGQkhYWF+h9fE0QpRWFhIZGRkZ09FI3mmOa4cQ1lZGSQnZ0dzODVaMB4QMjIyOjsYWg0xzTHjSGw2+3079+/s4eh0Wg0XY7jxjWk0Wg0miNDGwKNRqPp5mhDoNFoNN2cLplZLCL5wL4j3D0ZKGjH4XQV9HV3L7rrdUP3vfaWXHdfpVQDed0uaQjagoisDpdifbyjr7t70V2vG7rvtbflurVrSKPRaLo52hBoNBpNN6c7GoKnO3sAnYS+7u5Fd71u6L7XfsTX3e3WCDQajUZTl+44I9BoNBpNCNoQaDQaTTenWxkCETlTRLaLyE4Rua35PbomIvKciOSJyKaQtkQR+VxEMs3XhM4cY0cgIr1F5CsR2SIim0XkFrP9uL52EYkUkZUist687jvN9v4issL8vr8pIo7OHmtHICJWEfleRD40Px/31y0ie0Vko4isE5HVZtsRf8+7jSEQESvwBHAWMAK4XERGdO6oOowXgDPrtd0GfKGUGgx8YX4+3vACv1FKjQCmAjeaf+Pj/dqrgVOVUmOBccCZIjIVeAB4WCk1CCgGOrdOasdxC7A15HN3ue5ZSqlxIbkDR/w97zaGAJgM7FRK7VZK1QBvAOd38pg6BKXUN0BRvebzgRfN9y8CFxzNMR0NlFIHlVJrzfflGDeHdI7za1cGgdJzdvNHAacC88324+66AUQkAzgHeMb8LHSD626EI/6edydDkA7sD/mcbbZ1F3oopQ6a7w8BPTpzMB2NiPQDxgMr6AbXbrpH1gF5wOfALqBEKeU1uxyv3/dHgN8BfvNzEt3juhXwmYisEZHrzLYj/p4fN/UINC1HKaVE5LiNGxYRF7AA+JVSqsx4SDQ4Xq9dKeUDxolIPPAuMKxzR9TxiMhcIE8ptUZEZnbycI42M5RSOSKSCnwuIttCN7b2e96dZgQ5QO+QzxlmW3chV0R6ApiveZ08ng5BROwYRuBVpdQ7ZnO3uHYApVQJ8BUwDYgXkcDD3vH4fZ8OnCciezFcvacCj3L8XzdKqRzzNQ/D8E+mDd/z7mQIVgGDzYgCB3AZsLCTx3Q0WQjMM9/PA97vxLF0CKZ/+Flgq1LqnyGbjutrF5EUcyaAiDiBMzDWR74CLja7HXfXrZS6XSmVoZTqh/H//KVS6gqO8+sWkWgRiQm8B2YDm2jD97xbZRaLyNkYPkUr8JxS6t7OHVHHICKvAzMxZGlzgb8A7wFvAX0wJLx/qJSqv6DcpRGRGcASYCOHfcZ/wFgnOG6vXUTGYCwOWjEe7t5SSt0lIgMwnpQTge+BK5VS1Z030o7DdA3dqpSae7xft3l975ofbcBrSql7RSSJI/yedytDoNFoNJqGdCfXkEaj0WjCoA2BRqPRdHO0IdBoNJpujjYEGo1G083RhkCj0WiOUUTkElNI0C8iLapHLCI+U4xunYi0KEReGwLNMU3Il3qTiLwtIlHHwJhmisiJbTxGLxGZ33zPOvtcLSKPm++vF5Eft2UMmmML83v1Qr3mTcBFwDetOJTHFKMbp5Q6ryU7aEOgOdYJfKlHATXA9S3ZKSSztCOYCbTKENQfj1LqgFLq4sb6N4dS6j9KqZeOdH9N10AptVUptb1+u6kt9aCIrBKRDSLy87acRxsCTVdiCTBIRM419ea/F5FFItIDQET+KiIvi8h3wMsi0k9ElojIWvPnRLPfTBH5WkTeF5HdInK/iFwhhqb/RhEZaPZLEZEF5j/bKhGZborZXQ/82pypnBSuX7jxhF6IObZN5vurReQdEflEDC35v4f0+4mI7BCRlRiSCoQc+1bz/SDz97DevM7A+H8bcqMI1CiIFpH/mX03icilHfKX0nQ0PwVKlVKTgEnAtSLS39wWKSKrRWS5iFzQkoNp0TlNl8B8oj4L+AT4FphqCmv9DEN98jdm1xEYglwe0410hlKqSkQGA68DAT/rWGA4hlz3buAZpdRkMYrZ/BL4FYZuzcNKqW9FpA/wqVJquIj8B3ArpR4yx/Za/X7mseuMp5lLHIehlloNbBeRf2HUV7gTmACUYkgnfB9m31eB+5VS74pIJGARkdnAYAwNGgEWisjJQApwQCl1jjn2uGbGpelgRGQFEAG4gEQxVGQBfq+U+rSR3WYDY0QkMKuMw/h77wH6moJ0A4AvRWSjUmpXU2PQhkBzrOMM+cdYgqElNBR4UwxhLQfGlz/AwpCbrh14XETGAT5gSEi/VQHJXhHZBXxmtm8EZpnvTwdGyGH10lgxlE3r01S/hS0wAmAUFCk1x7MF6IshEbJYKZVvtr9Z7xoQQ3MmXSn1LoBSqspsn41xswgYDhfGjWIJ8A8ReQD4UCm1pAVj03QgSqkpEJTJuFopdXULdhPgl+EMRYgg3W4RWYzxgKENgaZL41FKjQttMJ+W/6mUWmj+8/w1ZHNFyPtfY2gtjcVwg1aFbAvVnvGHfPZz+P/CgjHzCN2PkBs+LehXUb9zI4SOx0fb/zcF+JtS6qkGG0ROAM4G7hGRL5RSd7XxXJqjz6fAL0TkS6VUrYgMwVBZdQCVSqlqEUnGcCf+vakDgV4j0HRN4jgsLTyvmX4HlVJ+4CoMUbbW8BmGmwgAc2YBUA7EtKBfW1kBnCIiSWLIa19Sv4NZiS074AsWkQjTJfYpcE1gZiIi6SKSKiK9MG4UrwAPAie001g1HYCIXCgi2Riy4v8TkcAM4BlgC7DWXGt6CuPhYTiwWkTWY7gS71dKbWnuPHpGoOmK/BV4W0SKgS+B/o30exJYIEaY5Se0/Ok8wM3AEyKyAeN/5RuMheIPgPkicj6GAWisX5tQSh0Ukb8Cy4ASYF0jXa8CnhKRu4Ba4BKl1GciMhxYZs5M3MCVwCDgQRHxm31/0dZxatoHpdRiYHG9tnc5rDQa2u7HUNb9Q71NS4HRrT23Vh/VaDSabo52DWk0Gk03RxsCjUaj6eZoQ6DRaDTdHG0INBqNppujDYFGo9F0c7Qh0Gg0mm6ONgQajUbTzfl/yabYiB7EGakAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(np.arange(100000,100050,1), reverse_top10fft[100000:100050], label = \"FFT top-10%\")\n",
+ "plt.plot(np.arange(100000,100050,1), concr[100000:100050], label = \"Original Parameters\")\n",
+ "plt.plot(np.arange(100000,100050,1), reverse_top10wv[100000:100050], label = \"Sym2 top-10%\")\n",
+ "\n",
+ "plt.title('Parameter Values') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.legend()\n",
+ "plt.draw()\n",
+ "plt.savefig(\"ParametersWaveletHaar.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 203,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0000, 0.0649, 0.0881, 0.0000, 0.0464, 0.0347, -0.0472, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, -0.0365, 0.0000,\n",
+ " -0.0422, -0.0344, 0.0372, -0.0823, 0.0000, 0.0764, -0.1654, 0.0000,\n",
+ " 0.0000, -0.0363, -0.0769, 0.0896, 0.0000, 0.0955, 0.0000, -0.0843,\n",
+ " 0.0000, -0.0387, 0.1598, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " 0.0000, 0.0000])"
+ ]
+ },
+ "execution_count": 203,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "top10_og[100000:100050]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 204,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACHdklEQVR4nO2ddXhUV/rHP2cmE3d3gQRKEkJwiluLLAXqQlu69a1uZbdUfvXutt1u3d2FGqVdWkpLKe4ElwQIRIm7Z87vjzsTJsnESCAy5/M8eSZz77l3zk1m5r3nle8rpJQoFAqFwnbRdfcEFAqFQtG9KEOgUCgUNo4yBAqFQmHjKEOgUCgUNo4yBAqFQmHjKEOgUCgUNo4yBApFL0cIMVkIkd7d81D0XpQhUHQrQohUIUSlEKJMCHFCCPGhEMK1u+fVGkKISCGEFELYddH5HIUQRUKIqVb2vSCE+KYrXkehaAllCBQ9gfOklK7AMGAE8FBHDhYavea93NSASCmrgK+Aq5uM0wOXAx+dudkpbJFe8+FR9H2klBnAz0C8EMJLCPGTECJXCFFo+j3UPFYIsUoI8ZQQYh1QAfQTQvxVCLFfCFEqhDgihLjJYvxkIUS6EOKfQogcIUSWEGK+EGK2EOKQEKJACPGAxXidEGKREOKwECJfCLFYCOFt2r3a9FhkWsmcbTrmWtPrFwohlgshIizOJ4UQtwohkoFkK5f/EXChEMLZYtsMtM/oz61dW1NMrxVt8fxDIcSTFs/nCCGSTKuQ9UKIBIt99wkhMkyvc1AIMa2l11H0HZQhUPQYhBBhwGxgB9p78wMgAggHKoFXmxxyFXAj4AYcA3KAOYA78FfgBSHEMIvxgYAjEAI8DLwDXAkMByYA/yeEiDKNvR2YD0wCgoFC4DXTvommR08ppauUcoMQYh7wAHAB4AesAb5oMt/5wGggtum1SynXA1mm4y2v73MpZV07rq1dCCGGAu8DNwE+wFvAUiGEgxBiIHAbMFJK6YZmiFI7+hqKXoiUUv2on277QfuiKQOK0L7MXwecrIxLBAotnq8CHm/j3EuAO02/T0YzJnrTczdAAqMtxm8D5pt+3w9Ms9gXBNQCdkCk6Vg7i/0/A9dZPNehrVQiTM8lMLWN+T4E/Gr63d10/NB2Xlu6xT4JRFs8/xB40vT7G8ATTc51EM3gRaMZnOmAobvfG+rnzP2oFYGiJzBfSukppYyQUt4ipawUQjgLId4SQhwTQpSguWM8TX5zM2mWJxFCzBJCbDS5eYrQVhe+FkPypZT1pt8rTY8nLPZXAuZAdQTwvcl9UoRmGOqBgBauIQJ4yWJ8ASDQVh9W52uFT4ApQohg4CLgsJRyRzuvrb1EAPeY52k6VxgQLKVMAf4OPArkCCG+NM1F0cdRhkDRU7kHGIh2x+7OSXeMsBjTIJ0rhHAAvgWeAwKklJ7AsibjO0IaMMtkoMw/jlKLY1iT7E0Dbmoy3klqLp9m87WGlPIYmkvpSjS30EeneG0VgGWsIbDJPJ9qMk9nKeUXpjl8LqUcj2YwJPBMa3NW9A2UIVD0VNzQ7tCLTEHaR9oYbw84ALlAnRBiFnBuJ17/TeApc8BXCOFnigNgeg0j0K/J+PuFEHGm8R5CiItP4XU/QvPTjwM+M23r6LUlAVcIIfRCiJlobh8z7wA3CyFGm7KtXIQQfxFCuAkhBgohppoMTxXa3994Cteg6GUoQ6DoqbwIOAF5wEbgl9YGSylLgTuAxWiB3SuApZ14/ZdMx/8qhCg1zWG06bUqgKeAdSb3yhgp5fdod89fmlxZe4BZp/C63wLewO9SyqxTvLY7gfPQ4i4L0OIJmM61FbgBLfBeCKQA15h2OwBPo/3NswF/4P5TuAZFL0NIqRrTKBQKhS2jVgQKhUJh4yhDoFAoFDaOMgQKhUJh4yhDoFAoFDZOl6gnnml8fX1lZGRkd09DoVAoehXbtm3Lk1L6Nd3eKw1BZGQkW7du7e5pKBQKRa9CCHHM2nblGlIoFAobRxkChUKhsHGUIVAoFAobp1fGCKxRW1tLeno6VVVV3T0VRQ/C0dGR0NBQDAZDd09Foeix9BlDkJ6ejpubG5GRkQhxqoKTir6ElJL8/HzS09OJiopq+wCFwkbpM66hqqoqfHx8lBFQNCCEwMfHR60SFYo26DOGAFBGQNEM9Z5QKNqmTxkChULRxRxZBTn7u3sWitOMMgRdiF6vJzExseEnNTWVVatW4eHh0bBt+vTp3HrrrSQmJhIbG4uTk1PDvm+++abR+ZYsWcK+ffu6bH6vvvoq0dHRCCHIy8tr2C6l5I477iA6OpqEhAS2b98OwMGDBxk+fDgJCQls2LABgLq6OqZPn05FRUWXzUvRQzHWw+KrYeWT3T0TxWmmzwSLewJOTk4kJSU12paamsqECRP46aefmo1PTU1lzpw5zY4xs2TJEubMmUNsbGyXzG/cuHHMmTOHyZMnN9r+888/k5ycTHJyMps2beJvf/sbmzZt4q233uKll14iMjKSO++8k2+//ZY33niDK6+8EmdnZ+svoug7ZO+CqmIoslqMquhDqBVBD2X9+vUsXbqUf/zjHyQmJnL48GGSkpIYM2YMCQkJnH/++RQWFgIwefJk7rzzThITE4mPj2fz5s1Wzzl06FCsaTT98MMPXH311QghGDNmDEVFRWRlZWEwGKioqKCiogKDwUBRURE//vgjV1999em8dEVP4ega7bHoePfOQ3Ha6ZMrgsd+3Mu+zJIuPWdssDuPnBfX6pjKykoSExMBiIqK4vvvvwdgzZo1DdsvvvhiHnzwwTZfb+zYscydO5c5c+Zw0UUXAZCQkMArr7zCpEmTePjhh3nsscd48cUXAaioqCApKYnVq1dz7bXXsmfPnnZfW0ZGBmFhYQ3PQ0NDycjI4NZbb+Xqq6+murqat956iyeeeIIHHngAnU7dP9gER1drj1XFUFkETp7dORvFaaRPGoLuwpprCGjRNdQRiouLKSoqYtIkrQ/5woULufjik73RL7/8cgAmTpxISUkJRUVFeHp6duo1w8PDWbVqFQApKSmkp6czaNAgrrrqKmpqanjiiScYMGBAp15D0UOpr4XjG8A1EMqytVWBMgRdyqs7XmWQzyCmhU/r7qn0TUPQ1p17X6RpmqQQghkzZnDixAlGjBjBu+++2+KxISEhpKWlNTxPT08nJCSk0ZgHH3yQJ598kpdffpnrr7+eyMhIHnjgAT777LOuvRBFzyAzCWrKYOT1sO5FzRAEJXT3rPoUn+3/jJGBI3uEIVBr/B6Mm5sbpaWlAHh4eODl5cWaNZrf9pNPPmlYHQB89dVXAKxduxYPDw88PDxYvnw5SUlJrRoBgLlz5/Lxxx8jpWTjxo14eHgQFBTUsP/PP/8kODiYmJgYKioq0Ol06HQ6lTnUlzn6p/aYuEB7VHGCLqW6vpqy2jLSStPaHnwG6JMrgr7CZZddxg033MDLL7/MN998w0cffcTNN99MRUUF/fr144MPPmgY6+joyNChQ6mtreX999+3er6XX36ZZ599luzsbBISEpg9ezbvvvsus2fPZtmyZURHR+Ps7NzovFJKnnzyyQZDc+ONN7JgwQLq6up44403Tu8fQNF9pK4B/zjwjQF7N2UIupjCKi3RI700HaM0ohPdfE8upez0DzATOAikAIus7HcAvjLt3wRENtkfDpQB97bn9YYPHy6bsm/fvmbbbIVJkybJLVu2dPc0eiy2/N44JWqrpHwiQMpl/9Sev3a2lJ9f3r1z6mPsydsj4z+Ml/Efxsvssuwz9rrAVmnlO7XTZkgIoQdeA2YBscDlQoimie/XAYVSymjgBeCZJvufB37u7FwUCkUXkL4V6iohcoL23DNcrQi6mILKgobfj5d2/9+2K9Yjo4AUKeURKWUN8CUwr8mYecBHpt+/AaYJU3RTCDEfOArs7YK52CSrVq1ixIgR3T0NRV8hdQ0gIHKc9twzXCsq01bvii6goOqkIUgvTe/GmWh0hSEIASwjHummbVbHSCnrgGLARwjhCtwHPNbWiwghbhRCbBVCbM3Nze2CaSsUCqscXaNlCDl5ac89w6G6BKqKunVafQmzIRCIPrMi6AyPAi9IKcvaGiilfFtKOUJKOcLPz+/0z0yhsEVqKyF9c4Nb6N3d7/KHLNf2KfdQl1FYVYiD3oFQt9AekTnUFVlDGUCYxfNQ0zZrY9KFEHaAB5APjAYuEkI8C3gCRiFElZTy1S6Yl0Kh6Chpm6C+BqImsjZjLS9tf4mzveOZAqZagiHdPcM+QX5VPt6O3oS7hXO8pPsNbFcYgi1AjBAiCu0L/zLgiiZjlgILgQ3ARcBKUwR7gnmAEOJRoEwZAYWiGzm6BoSeyuChPLlc05Q6WpWj7VMrgi6joKoAb0dvwtzC2JW7Cyllt/bO6LRryOTzvw1YDuwHFksp9wohHhdCzDUNew8tJpAC3A0s6uzr9kTS09OZN28eMTEx9O/fnzvvvJOamhqrYzMzMxs0hFpj9uzZFBUVndJ8Hn30UZ577jmr20NCQhpE6pYuXXpK5+9q/vWvf3X3FBRHV0PwUN48+BkZZRmMCx5HdkUOFY7uUKhUSLsKsyEIdw+ntLaUouqibp1Pl8QIpJTLpJQDpJT9pZRPmbY9LKVcavq9Skp5sZQyWko5Skp5xMo5HpVSNv/W6iVIKbnggguYP38+ycnJHDp0iLKyMqsCc3V1dQQHBzfrP2CNZcuWdVozyBp33XUXSUlJfP3111x77bUYjcZ2HVdfX9/lczFzKobgdM7H5qgug8ztHApN4OO9H3N+9PlcOOBCAFI9Q9SKoAuxXBEA3R4n6O5gcZ9h5cqVODo68te//hXQmtS88MILvP/++1RUVPDhhx8yd+5cpk6dyrRp00hNTSU+Ph7QlEMvueQSYmNjOf/88xk9ejRbt24FIDIykry8PFJTUxk0aBA33HADcXFxnHvuuVRWVgLwzjvvMHLkSIYMGcKFF17YIemHQYMGYWdnR15eHvPnz2f48OHExcXx9ttvN4xxdXXlnnvuYciQIWzYsIHHH3+ckSNHEh8fz4033mguCmTy5MncddddjBgxgkGDBrFlyxYuuOACYmJieOihhxrO9+mnnzJq1CgSExO56aabqK+vZ9GiRQ3qrQsWLGhxnLX5LFq0iNjYWBISErj33ntP9V+oOL4Ro7GOxypTcLN34+7hdxPlHgXAUVcvZQi6CCklBZUFeDtpMQLo/lqCvikx8fMiyN7dtecMHAyznm5x9969exk+fHijbe7u7oSHh5OSkgLA9u3b2bVrF97e3qSmpjaMe/311/Hy8mLfvn3s2bOnQbK6KcnJyXzxxRe88847XHLJJXz77bdceeWVXHDBBdxwww0APPTQQ7z33nvcfvvt7bqsTZs2odPp8PPz4/3338fb25vKykpGjhzJhRdeiI+PD+Xl5YwePZr//ve/AMTGxvLwww8DcNVVV/HTTz9x3nnnAWBvb8/WrVt56aWXmDdvHtu2bcPb25v+/ftz1113kZOTw1dffcW6deswGAzccsstfPbZZzz99NO8+uqrDeqt+/fvtzru6quvbjSf/Px8rrvuOg4cOIAQ4pTdaAogdTXfuHuwqzSVf43/F56OnjgbnNEJHUcdHKFot1ZLoPpAd4ry2nJqjDX4OPoQ4haCQDReEeQcgNpyCBne8km6mL5pCHoo55xzDt7e3s22r127ljvvvBOA+Ph4EhKsqzxGRUU1GInhw4c3GJM9e/bw0EMPUVRURFlZGTNmzGhzLi+88AKffvopbm5ufPXVVwghePnllxt6KKSlpZGcnIyPjw96vZ4LL7yw4dg//viDZ599loqKCgoKCoiLi2swBHPnamGhwYMHExcX1yBe169fP9LS0li7di3btm1j5MiRgNbDwd/fv9n8fv/99xbHWc7Hw8MDR0dHrrvuOubMmcOcOXPavHaFdXKPruJFb09GB45iTj/t72ivtyfUNZSj0gg1pVBZCM7N38OK9mOuIfB29MZB70CASwBpJSZDkLYFPjkfPELh1o1nbE590xC0cud+uoiNjW3m8y8pKeH48eNER0ezfft2XFxcOvUaDg4ODb/r9foG19A111zDkiVLGDJkCB9++GFDD4HWuOuuuxq5UVatWsVvv/3Ghg0bcHZ2ZvLkyVRVVQGaoJ1erwegqqqKW265ha1btxIWFsajjz7aMM5yjjqdrtF8dToddXV1SClZuHAh//73v1udX2vjLOdjZ2fH5s2b+f333/nmm2949dVXWblyZZvXr2hCVTHP1GVQ7eDKQ2MeapTBEuURxdH8g9qTouPKEHQSS0MAEOYWpq0I0rfCpxdoBrck84zOScUIuohp06ZRUVHBxx9/DGhBzHvuuYdrrrmmzf6+48aNY/HixQDs27eP3bs75tYqLS0lKCiI2traU+4PUFxcjJeXF87Ozhw4cICNG63fjZi/9H19fSkrK2tXwNuSadOm8c0335CTo6UkFhQUcOyYlo1iMBiora1tc5wlZWVlFBcXM3v2bF544QV27tzZofkoNFYnvcdyF2duiJhNpEdko31RHlEcq86nHlT/4i4gvyofOGkIwt3COV50RFsJOPvAiOugulgr7jtDKEPQRQgh+P777/n666+JiYlhwIABODo6tisT5pZbbiE3N5fY2Fgeeugh4uLi8PDwaPdrP/HEE4wePZpx48Zx1llnndL8Z86cSV1dHYMGDWLRokWMGTPG6jhPT09uuOEG4uPjmTFjRoPrpr3Exsby5JNPcu6555KQkMA555xDVlYWoElcJyQksGDBglbHWVJaWsqcOXNISEhg/PjxPP/88x2/eBunsq6SfyV/Sb/aOq4dc3+z/ZHukdQYa8m006uAcRfQdEUQKnUU1JZQ7uwF1/wEIcO0gaXZZ25S1iRJe/pPX5Ohrqurk5WVlVJKKVNSUmRkZKSsrq7u5ln1HXrze+NM8NqO12T8h/Fy84fTrO7flr1Nxn8YL//8b5SU/7v3DM+u7/Fm0psy/sN4WVNXI2XGdrn8hSgZ/2G83J/6hzbg0AopH3GX8tiGLn9tWpCh7psxgl5GRUUFU6ZMoba2Fiklr7/+Ovb29t09LYUNkFGWwft73mNmWTkj48+1OibKw5RC6ubDRLUi6DQFVQW42bthOLEXPp5PmIs7AGnUchaAqyl5ouzEGZuTMgQ9ADc3t4a6AYXiTPLfrf9FJ+GegiIItS5l7uXohaeDJ0d1QrmGuoCCqgJ8HH3g62vAwY2wSxfDz5ec1BxyC9QeS8+cIVAxAoXCRtmYtZEVx1ZwvWc8gfX1EJTY4tgojyiO6k2GQPUl6BQFVQV4G1yh8CiMuQVX/0F4O3qfrCVw9gWhh7IzFyNQhkChsEFqjbU8s/kZQlxDWFhh1HoOtJIWGuURRaqxEmrKoKKgxXGKtimoKsDbbEuDEwGLFFIAnU5zD6kVgUKhOJ18deArUopS+OfIf+KQvavV1QBAlHsUBfWVFOl0KoW0kxRUFeBdUw0ICNSKRxsZAgDXALUiUCgUp4/8ynxeT3qdscFjmeIzBApTIXhoq8eYA8apBjsVJ+gE9cZ6CqsK8a4oBL+B4OAKaLUE2eXZ1NSb1IrdAtWKoLfy1FNPERcXR0JCAomJiWzatKnT51yxYgXDhw9n8ODBDB8+vMWq2RdffLFDYnNtce211+Lv798gjGemoKCAc845h5iYGM455xwKCwsB+Pbbb4mLi2PChAnk52sFM4cPH+bSSy/tsjkpuoZXdrxCZV0l9426D5G9S9toclG0REPmkMGgDEEnKKouQiLxLs5qZHxD3UKRSNLLTP2L1Yqgd7JhwwZ++umnBmG53377jbCwsLYPbANfX19+/PFHdu/ezUcffcRVV11ldVxXG4JrrrmGX375pdn2p59+mmnTppGcnMy0adN4+mlNzuOVV15hy5Yt3HTTTXz++eeAJoD35JNPdtmcFJ1nb95evkv+jgWDFtDPox9kJWk72nANBbsGY9AZOOrkqgxBJzAXk3lVFjcyBOHumgppg+aQWyCU50F93RmZlzIEXURWVha+vr4N+jq+vr4EBwezcuVK5s+f3zBuxYoVnH/++YAmp/yPf/yDuLg4pk+fzubNm5k8eTL9+vVraBYzdOhQgoODAYiLi6OyspLq6upGr/3yyy+TmZnJlClTmDJlCgBffPEFgwcPJj4+nvvuu69hrKurK3fddRdxcXFMmzaN3Nxcq9czceJEqwJ5P/zwAwsXLgRg4cKFLFmyBNC0hKqrq6moqMBgMLBmzRoCAwOJiYnp6J9ScZowSiP/3vxvvB29uXnIzdrGzKQ2A8UAdjo7ItwjOOrkcmqGwGiEpM+hvrbjx/YhzIbAp76+kSFo1pfANQCQUJ5zRubVJ+sIntn8DAcKDnTpOc/yPov7Rt3X4v5zzz2Xxx9/nAEDBjB9+nQuvfRSJk2axJQpUxokJPz8/Pjggw+49tprASgvL2fq1Kn85z//4fzzz+ehhx5ixYoV7Nu3j4ULFzYoeZr59ttvGTZsWCMxN4A77riD559/nj/++ANfX18yMzO577772LZtG15eXpx77rksWbKE+fPnU15ezogRI3jhhRd4/PHHeeyxx3j11fZ3Bz1x4kSDomhgYCAnTmh+zPvvv5/p06cTHBzMp59+ysUXX8yXX37Z7vMqTj+f7/+cnbk7eWLcE7jaa75pMne0GR8wE+URRXLx8VMLFh9dBUv+pmnpDGhbHbev0iAvYQQCTrpdvRy8cDW4nuxL4BqgPZadAPfg0z4vtSLoIlxdXdm2bRtvv/02fn5+XHrppXz44YcIIbjqqqv49NNPKSoqYsOGDcyaNQvQtPtnzpwJaLLNkyZNwmAwMHjw4Eb9CkDrd3Dffffx1ltvtTmXLVu2MHnyZPz8/LCzs2PBggWsXr0a0O7czX77K6+8krVr157yNQshGlQqzznnHLZt28aPP/7IDz/8wOzZszl06BAXXXQRN9xwQ5e6rRQdJyknif9u/S+TwyYzt7/pBqOiQPtSb8MtZCbSPZI0WUvtqdQSnNinPZZbX4HaCg2GwLMf2J8UoxRCNM4cOsNFZX1yRdDanfvpRK/XM3nyZCZPnszgwYP56KOPuOaaa/jrX//Keeedh6OjIxdffDF2dtqf3WAwNHyRWso2myWbzaSnp3P++efz8ccf079//y6dsxCCtLS0hn4CN998MzfffHOL4wMCAsjKyiIoKIisrKxmvQTM3diWL1/OnDlz+O677/jmm2/47LPPGprnKM4shVWF3PvnvQS4BPDkuCfRCdP9X5ZJqbWNQLGZKI8o6pGkUUO/inxw8W3/JHL3a48V+e0/pg+SX5mPTko8goY12xfmFsbBQpPcd8OK4MwEjNWKoIs4ePAgycnJDc+TkpKIiIgAIDg4mODgYJ588smGVpbtpaioiL/85S88/fTTjBs3rsVxbm5ulJaWAjBq1Cj+/PNP8vLyqK+v54svvmDSpEkAGI3GBunozz//nPHjxxMWFkZSUhJJSUmtGgHQGs989NFHAHz00UfMmzev0f7//Oc/3HHHHRgMBiorKxFCoNPp1Iqgm6g31rNozSIKqwp5fvLzeDhYqNq2M1Bspp9HP8CcOdRB91CO2RDYdjFaQUkaXvVGdCHN3XFhbmFklGVQZ6w7aQjO0IpAGYIuoqysjIULFzb0zt23bx+PPvpow/4FCxYQFhbGoEGDOnTeV199lZSUFB5//HESExNJTExs0Oi35MYbb2TmzJlMmTKFoKAgnn76aaZMmcKQIUMYPnx4wxe2i4sLmzdvJj4+npUrVza0nGzK5Zdfztlnn83BgwcJDQ3lvffeA2DRokWsWLGCmJgYfvvtNxYtWtRwTGZmJps3b24Ijt9+++2MHDmSN998kyuuuKJD163oGt7e/TbrM9ezaPQiYn1iG+/M3AGeEe1uNGPuU9DhFFKjUWu/CDa/IigoPoa3sR6Cm68Iwt3DqTPWkV2eDXb24OR95lJIrUmS9vSf3ihDfeutt8p33323u6chXVxcunsKZ5ye/t44XazPWC8HfzhY3r/6fmk0GpsPeGGwlF9d1aFzTv1qinzglUgp17zQ/oMKjmqyyo+4S/nFFR16vb7Ggs8nyuveiJGyprLZvs1Zm2X8h/FyfcZ6bcNrY6T8/PIufX1akKFWK4IzwPDhw9m1axdXXnlld09FYSNkl2dz3+r76O/Zv1nrSaDDgWIzUZ79OOrg1LEVgdktZHCx+RVBYU2pJjhncGy2z2oK6RlaEfTJYHFPY9u2bd09hQbKysq6ewqK00ytsZZ//PkPquqr+O/k/+JssNIqtYOBYjOR7pH8z7AJWXgM0fZwjRxTxlDYqDPei7dHISUFxhp8XKwXmvo7+2Ovs2+cOZSXbHVsV9MlKwIhxEwhxEEhRIoQYpGV/Q5CiK9M+zcJISJN288RQmwTQuw2PU7tzDykksdVNMHW3hO19bU8uPZBknKTeGzsYw0B3mZk7tAeO7oi8IiiTEBeSUdWBAfAIwy8Im16RVCdn0yZTuDtEWF1v07oCHMLO9mXwDVAqyM4A+/hThsCIYQeeA2YBcQClwshmkSluA4olFJGAy8Az5i25wHnSSkHAwuBT051Ho6OjuTn59vcB1/RMlJK8vPzcXRsvgzvi1TUVnDbytv4+ejP/H3Y35kVNavlwVlJHQoUm2nQHKrIbv8XVM5+8B+kvVZlgRY8tkEK09YD4O0zsMUxYW5hpJVZrAiMtVBZeNrn1hWuoVFAipTyCIAQ4ktgHrDPYsw84FHT798ArwohhJRyh8WYvYCTEMJBStlYQ6EdhIaGkp6e3qJkgsI2cXR0JDQ0tLuncdopqCrg1t9uZV/BPh4f+zjnx5zf+gGZSR12C4FFCqnOyKjyPHD1a/2A+jrIOwj9p2hVxdII1cXg5NXh1+7t5GdqLmJv/7gWx4S5h7EpexNSSoS5ZWVpdocNdkfpCkMQAlgIaZMOjG5pjJSyTghRDPigrQjMXAhsb8kICCFuBG4ECA8Pb7bfYDAQFRV1ipegUPReMssyuWnFTWSVZ/Hi5BeZEj6l9QPMgeLh13T4tfyd/XHS2Z+sJWjLEBQcgfoa8I8Fc8C6osAmDUFBnnZv7O0S0OKYMLcwKusqyavMw8/VVF1clg0BTZ0sXUuPyBoSQsShuYtuammMlPJtKeUIKeUIP7823nwKhY2QXJjMVcuuIr8qn7fPebttIwAnC8naqTFkiU7oiHQN4ajBrn1FZeZAsf8gLS8ebLOozGikoDAVAG/Hlu/uw920m9zjpcfPqMxEVxiCDMAyDB5q2mZ1jBDCDvAA8k3PQ4HvgaullIe7YD4KhU2QlJPEwl80JdgPZ37IsIDmRUpWyUzSHoOGnNLrRnnFcNS+nUVlOfsBoTVhcfbRttliwLjwKAUmZ4ePo0+LwxqlkJ5BmYmuMARbgBghRJQQwh64DFjaZMxStGAwwEXASimlFEJ4Av8DFkkp13XBXBQKm0BKySPrH8Hd3p1PZn/CAK8B7T/4FAPFZqK8Ysiys6Oi8Gjbg3P3g3c/MDiBs8kdVGmDK4LMHRTo9Djq7HGyc2pxWJBrEHqh1zKHHFzB3rV3rAiklHXAbcByYD+wWEq5VwjxuBDCrKP8HuAjhEgB7gbMKaa3AdHAw0KIJNOPPwqFolU2Z2/mSPER/jbkbwS7dlCm+BQDxWbMmUPHitqxgDdnDIFtrwgyd1BgZ8Dbyad5cZ8FBp2BIJcg0kvPbKeyLikok1IuA5Y12fawxe9VwMVWjnsSUC2sFIoO8uWBL/F08GRm1MyOHWgOFI/omPihJQ0ppCXHaVU5q7YK8g9DrEmY0MEddHa2GSPITCLf2QPvVtxCZsLdw0kuStYyh85Q7+IeESxWKBTtJ7s8mz/S/uD8mPNx0Du0fYAlHVQctUaEewQCOFpbDLmHWh6Ynwyy/uSKQAgtYGxrKwKjEbKSKDA44u3UtjtuathUUopS2Hpi68mistOMbRmCVc/A1g+6exYKRaf4+tDXGKWRSwZc0vGDOxkoBnDQOxDiHEiqwQ4O/q/lgWaNIX+L1EdnH9szBPkpUFNGgU62mjFkZl70PLwdvXl/z/ta5tAZMAS2pTV04Cet7VsnlsUKRXdSW1/Lt4e+ZWLoRELdTqFQroOB4nfXHGFfVglBHo4EejgR5O5IoIcjoe79OVyWCwf+B+Pvsn5wzj7QGcDbopmSs/cZqZTtUWTuQAIFdZXtMgSOdo4sGLSAV3a8wsHQ+QysKYPqMi14fJqwLUPg4gvleW2PUyh6EhUFYO8Cdg6sOLaC/Kp8Lh146amdK3OHVS18axRX1PLMLwdwtNNTWVtPnfGkpIS9nwOOPvWUZ2zFpTT7ZM67JTkHwDdG09Y34+x9xoTUegyZOyizd6FW1rXLEABcOvBS3tv9Hh+UJfM0aKsCZQi6CBc/LXjVhJKqWsqr6wjyaDmtS6HoFqSEtyeB3gEu/5IvD35JmFsY40Ja7lbXIhUFWu7/iGvbNfy3/SeorZcsvmkUCaGe5JdVk1VcRVZxFSuOlvFrwUp2Odhz9sFl1s+Zsw9Chjfe5uRte8HizB0UBAwCctptCDwcPLhowEV8tu9TbrfTE1KaDT5d26bWEtuKEbj4NVsRHM0rZ9aLa7jg9fVKsE7R8yjJ0L6885M5+MF0duTs4NKBl57sO9wRjq7WHttZUbxsdxbBHo4khnmi1wn83R0ZEubJzPhAHpvxF0Dwh6MvZTt/aH5wdZmWneTfRBrB2UerI7CVz5qxHrJ3UeCraTS1VkzWlKtir0IIwcfu7qc9hdS2DIGzD9SWQ43WP3dvZjEXv7mezOJKsoqrSM5RWv2KHkb2bu3xgnf4wtsbB6NkfkFOx79IjUZY/Rx4RUHE+DaHl1TVsiY5j1mDg6zmvbvauxLtOYB1jh44pK2lrqKo8YBcUxN2/yYJps7eYKyD6pKOzb+3UpIBtRUUuGvlUe3JGjIT6BLI7PBz+M7NhcKi1NM0QQ3bMgQuJo2iijy2phZw2dsbsdfr+OCakQCsT1Hxg55AZlFld0+h55C1CxCURI1nmYOe2QYfPH5/Er6/WcvTby/7l8KJ3TB5Eejb9giv3J9DTb2R2YOt+P5NjAwcxgnnGqCOVT993ninpcaQJbZWVFZVDEC+yZZ6OXRMbO/aITdRpdPxZc7Grp5ZI2zSEGzbd4gr39uEn6sDX/9tLJMH+hPq5cSGIzby5uzBbE0tYOzTK9mSamN+5JbI3gXe/fgh7Xcq66u4bMbrMPkB2PUlfDgbSrLaPoexHlb9G3wHwOBmdZ1WWbY7i0B3R4aGtfzFNcx/GNWyhu1O3lTt+ZEjuRYr6pz9YOekNaOxpMEQ2EjmkMkQFFAPtC44Z43+XtFMrpF8XpZCRW1Fl0/PjI0ZAl8A3ly2kf5+riy++WxCPLUA8dn9fNh4pACj0UZ8lz2UFfu1nOnNR5UhACB7F8bAwXx18CsS/BKI9YuDyffBpZ9qWTkfzm74smmRPd9B7gFtNaDTt/mSZdV1rDqUy8z4QHS6luUQEv0TAdgbnsBk3Q4e/Gbryc9P7n5NaK7p6zUokNrITVeV5gIrMFbjZu+GQW/o8CmuFV4UyTq+T/m+q2fXgE0Zgh+SawBI9K7nixvH4Ot6sipzbLQPxZW17MuyEd9lD2X1Ic09tzu9jS83W6CyEIqOs9HDh2Mlx7hs4GUn9w06D678BgqPwdLbW44Z1NdpqwH/OIhto1mNiZUHcqipMzJ7cFCr4wJdAglxDWGPqwuuVGJIW8+nm0zS1JYaQ5Y425ohMK0I6iqbBYrTCioorqxt8xRDXcMZWq/j470fU2tse/ypYDOGoKbOyIc7ywG4cbg77o6NLfPZ/bTVwobDNvIG7YHklFaxP6sEnYDdGcoQkL0HgC+qs/B29GZG5IzG+yPGwvRHYN8PsPlt6+fYvRgKDsOU+0HXvo/7z7uz8HNzYHhE2/7sof5D2V6RgdHgwkKvvTzz8wEysjKgNKt1Q2ArCqSmoHhBXXkjt1C9UXL+6+t44PvdbZ/D1Z9rS8rJLM9keery0zJNmzEE9nY63r9hEtLOCUNV86BwoIcj/XxdVJygG1ljWg2cNySYjKJK8ss63LG0b5G9i2SDgT8L93JhzIXY6+2bjxl7BwyYBcsfhPStjffV18Kfz0BgApw1p10vWVFTxx8Hc5gZF4i+FbeQmaH+Q8mvKiC9/wQmswUw8v73P2s7m6aOAjh4gNDb3oqgpqSRIUhKKyKvrIYVe09QXNHGXb5rIBOLcunv0Y8P9nxwWtLcbcYQAHi52CNc/Fp8E47p78PmowXU1dtmc+3uZk1yLj4u9lw6QmvOYfOrgqxdvOXrj5OdE1fFXmV9jBBw/hvgHgRfX9O4WCvpMyhMhSkPnmwT2QarDuZSVdu2W8jM8ACtYGx7QDT68hM8NbKW6gxtJWN1RaDTaW0qbaWorKoYDM4UVBU2MgR/HswBoKbeyE+7M1s/h1sAOuCv/eZTa6wlr7LrsxttyhAAJpkJ6w3ux/b3oay6Tn0BdQNGo2RNch7jY3wZHOoBqDhB8okkfnXUc8WgK/BybMVN4+QFF3+kyRB8f7NWM1BXDX/+B0JGwIAZLR/bhGW7s/BxsWdUVPuyW6I8ovBw8GCHvh50dpyr38pAXTrVehdwD7F+kC0Jz1UVU+foQVF1UaMaglWHchke4UWMvyvfbW/a0LEJpt7Fc7ziWDJvCX7OXd+qVxkCC8b004I561Wc4IyzL6uE/PIaJsb44eZooJ+fC7ts2SDXVvGWzMNJ2LEwdmHb40OGwYx/QfJyWP8SbP8YStJhygPtXg1U1daz8kAOM+Lb5xYCrYfxUL+hbM/fC5HjcT7yC8OcskkhvOXXtSXhuapiihzdkJxUHs0rq2ZXejGTB/hxwbBQth0r5Fh+ecvncNNaVurLck6torwd2KAh8INy61/0vq4ODAxwY6OKE5xxVidrxnlCjBa0TwjxYI8NG4Lkoyv41dmJBQFj8XT0bN9BI6+H+Avh9yfgj6cg/GzoP7Xdr7nqYC4VNfXMjm+fW8jM0IChpJakUhA9FfIOcVbtAXZWB3I0r4UvNxtbERQ4uAAnawhWH9Le65MH+jN/aDBC0PqqwLQiOJ0yEzZoCEwrghYCLmf392FLagHVdfVneGK2zepDuZwV6Ia/uyMA8SEeZBVXkVPagerZPsSbez/ESUquTri+/QcJAee9BN5R2h13B2IDAD/vycLL2cCYfh0rehrmr6mZ7vDSDIhe1nJIhrF8bwtfXM42JDxXXUKhg1arZDYEfxzMxdfVnrhgd4I8nBjb34fvd2S0HAR28QMElOWctmnaniFw9oX6aqgutbr77P4+VNUa2Zlmu3ejZ5ry6jq2HStk0oCTvs+EUE8Am1wVJBcm82vJIRaUV+MZ0MEGMg5ucOW3cME7EDWh3YdV1dbz+/4cZsQFYqfv2NdCrE8s9jp7dpQdbxC0q/U5i1/2tGAIzF3KbEF4rqqYAoOW7eXj6EO9UbImOZeJA/waivXOHxrK8YIKth1rwV2mt9OMQalaEXQdFnpD1hgT5YMQsP6w0h06U2w8kk9tvWSihSGIC3ZHCNidbnsFfm/ufBMXKbjaOarduf+N8IqEhI51L1ubnEdZdR2z2pktZIm93p5433h25OzQ+hPr7OgXN4qktCKyi62s6Jx9wFgLNTYg8lhVTIFJ28nb0ZuktCKKKmqZPNC/YcjM+ECcDHq+29GKe8jt9LastF1D0EKDGg9nA3HB7qqwrAuoN0qW7Mhg6c7W0+NWH8rFyaBnROTJzBgXBzv6+7myO6PoNM+yZ3Go8BC/HvuVK0rL8QxMPGOvu2xPFh5OBsb2b79MsiXDAoaxL38flSOvg5vXMmmoljr66z4rd7G2Ul0sJVSVkK/XoRd63B3c+fNgDjoBE02xMABXBztmxgfy085MqmpbcEm7BqgVQZfiYnqjt5A5BJru0I7jRS3/UxStIqXklz3ZzHxxNX//Kom/f7mj1VTQ1cl5jOnnjYNdY12ahBAPdtlYCumbO9/Exc6JqwsLtEKw00RVbT1JaUV8uvEY93+3m593Z3NubACGDrqFzAz1H0qdrGN3wQHwH0S0vyvR/q7W3UO2okBaWwnGWgow4uXohU7oWHUol8QwTzydGxcHnj80hJKqOlYeaCEO4Hp6exfboCEwrwhaNgRj+/tSU29s2WensIqUmv9z3mvruPnTbRil5IVLh+Dj6sCi73ZZLdRLK6jgaF45E2Ka50YPDvUgp7SaEyW9L2CclFbEa3+kdKgK9FDhIVYcW8EVviPwNBohqGsNQU5JFf/8ZifnvvAncY8sZ/5r63hoyR6W7c5ieIQXt0yJPuVzD/EbgkCwPWd7w7YZcQFsOlpAYXlN48ENwnN9/PNllpdAa1HZkDZq4RYyMy7aF383h5azh9wCtGCx8fTcnNpWq0rQgsXQau/ikVHe6HWC9YfzGBft2+I4xUn2Zhbz5E/72XAknxBPJ569KIELhoZgp9dhr9dz6+fb+WBdKjdM7NfoOHPaqDk+sCNnB6/ueJXHxz3O4JCThWUBsY5n9oI6yet/pPDrvhP4uTlwialSui3e3PkmLgYXFuIBOjvwO6vL5lNQXsOCdzeRVljB2P6+zIwLJDbYg7hgd0K9nKw2n+kIHg4eRHtFa3ECEzPjgnjtj8P8tv8EF1v+DWxlRWCWlzDW4O3o35A2OsWKIdDrBOcPDeG9tUfJL6vGx0IQE9BWBLJe+5u5Nj++s3TJikAIMVMIcVAIkSKEWGRlv4MQ4ivT/k1CiEiLffebth8UQrS/BPJUMTiCg3urhsDVwY6EUA8VJ+gAN3+6jYMnSnnkvFhW3juJS0aENWSfzB4cyLSz/Hl+xSHSCkya6ru/gaeCiF5zF3Pdkunv68TmrM3ctOImNmdv5rvk74gNdkcn4OCxNNjxGXxzLWRs68arbB919caG984TP+5rV6OdjLIMbTVw1hV45BwEv0Fg59Dmce2hpKqWq9/fxPGCCj64ZhTvXzOSu88dyMz4QMK8nTttBMwM8x9GUk4SdcY6AOJD3AnxdGqeRmorwnNmQ1BfibejN6ss0katcf6wEOqMkp92WekxYSoqqypsQ47iFOm0IRBC6IHXgFlALHC5EKKp2tR1QKGUMhp4AXjGdGwscBkQB8wEXjed7/Ti7NOqawi0OMHO9GLKqutO+3R6O2XVdaQVVHLd+Cj+Oi6qma9fCMHj8+MRAv7vhz2au2T/UqTQM6h0Iy/XPsK61xO5ZcWNhDj5M9h3ML8e/RmnvV/xhcvz3LRpBvxwC+z5FnZ+2U1X2X52phdRWl3HollnUS8l9327q00X0ep0rZ/w3P7nac1ousgtVFFTx7UfbOFgdilvXjmcs08xGNwehvoPpaKuguTCZED7v8+IC2S1KSOpAUdPEDobWBGYXEO15Xg5eLO6SdpoU84KdCc2yJ3vtqc32i6lZE+JVovwjw9+PS2xy65YEYwCUqSUR6SUNcCXwLwmY+YBH5l+/waYJrTbkHnAl1LKainlUSDFdL7Ti4tfi+mjZsb296XeKFWnrHZg7kzV38+1xTEhnk7cc+5AVh3M5aedmXBsPflh0xlZ/RrvDvkbd7hBVFUl7+9ZzwVZh0ktTePgz3cSI9L4XMxGXv87hI6EE3vP1GWdMqsP5SEEXDYyjAdmD2JNch6fbTre6jFr0tcQ5hZGhHDUblICB3d6HlW19dz48Ta2Hy/kpcuGMuWsrncpWGIuLGsaJ6ipM/LnQYsbL1sRnqsqokoIyuurqK5xbpY2ao0LhoWwM72YlJwyKmrq+HzTcWa9tIabf9BiBzMiBNV1XS+K2RWGIARIs3iebtpmdYyUsg4oBnzaeSwAQogbhRBbhRBbc3Nbv5tvExe/Vl1DAMMjvDDohXIPtYMjuZqUQH8/l1bHXTM2koRQDz768Vcoz2U7sRjdD/Bq6XLO8h3Mu/O+w2vCP5iGK3oEy8dez4+Tl/Nw5WVku8VBQDyc2NPjC5HWJOeSEKplhiwYHc6EGF/+tWw/x/OttxqsqqtiS/YWxoeMR5ib1XcyY6i23shtn+9gbUoez140pN1qop0hyDWIQJfARnGCEZHe+LjY80tT95C5qKwvU1VMkakOJDNf1yxt1BpzE4PRCbh7cRJj/vU7D3y/GyEEd84bD8CcKIGHU8e7nLVFr8kaklK+LaUcIaUc4efXSfU9l7ZdQ072eoaGeylD0A4O55ahExDu49zqOL1O8K/zB3NW9S4A3igrxTH4C4b4JfD2OW/jERAHU+7H6/rfGR18NssLdhFvUiLdlV4MAXGa37WkDbXGbqS4spaktKKGD7wQgmcuTEAvBP/4ZqfVVqhbT2ylqr6KCSETIHuntjEw/pTnUG+U3LN4J7/tP8Hj8+K4aHjoKZ+rowz1H8qOEzsaXGF6neDcuABW7j/R2KXh7NP3YwTVJZSYDEFKdr3VtNGm+Ls5Mm1QAHszS5gwwI/FN53NsjvGc/GYGHD0OG0ppF1hCDIAy7SIUNM2q2OEEHaAB5DfzmO7jHd2vcPSw0upd/bV7kaMrS+xzu7nw57MYtUgpQ2O5JYT7u3cLDZgjfgQDy73T+MTFz+SnX4g2DGeN6a/gat9Y7fSzMiZpJWmoXPMQK8TWh1CgOnL8cS+03EZXcKGw/kYJY3SYYM9nfi/82LZdLSAD9enNjtmbcZaHPQOjAwcCdm7tcpgR49TnsOrK1NYujOTf84cyNVnR57yeU6F0YGjyanMISk3qWHbjLhAymvqG1fr24LeUFUxZXba3fuxXGObbiEzL12WyJYHp/PaFcMYFeV9Mph/GovKusIQbAFihBBRQgh7tODv0iZjlgJmLd2LgJVSu2VYClxmyiqKAmKAzV0wp2bUG+tZlb6KB9c+yEW5K1npaEC2IYU7e3AQUsKSpNMTqe8rHM4to18r8YFGSEkWe/iPnxP15f15bPTzOBuarySmhk/FTtjxR/oKYvxdtR4RAaYchBN7unD2Xcua5Fxc7PUMDfdstP3i4aFMO8ufZ5cfaIipNByTvoaRgSNxtHOErF2dcgulF1bw+qoU5iQEccvkU68LOFVmRc3C08GT93a/17BtbH9f3BzsWL7H4m7W2RZcQyWUOmifC2O9I5MHts+T4Wxvh7eLlZWD6+mTmei0ITD5/G8DlgP7gcVSyr1CiMeFEHNNw94DfIQQKcDdwCLTsXuBxcA+4BfgVinlaamY0Ov0fDLrE56b9Bx1QnBngB9X/nYjW7K3tHjMwEA3hoR68PXWtNPSHq4vUG+UHM0rbzM+YGbjoe/5p7sdA+z9Odd3ESMjAqyO83DwYEzwGH5N/ZXBIe7szihGOriDR1iPDhivSc7j7P4+zSp0hRD8+4LBONjpuefrndSbXETHSo5xvPQ440PGa1kmhUcbGQIpJU//fIBlu62kFFrhX8v2IwQ8MNtKd7AzgLPBmQWDFvBn+p8cKjwEaG1ipw7yZ8X+EyeLCp19tBVBX/5cVRVTaq9l+3g6uhEffOqrPADcAnv0igAp5TIp5QApZX8p5VOmbQ9LKZeafq+SUl4spYyWUo6SUh6xOPYp03EDpZQ/d8V8WkIndMyInMH3Ix7i0dx8sityuHb5tdz8280NKW9NuXhEGAeyS1XXshbILKqkus7YrhXB7tzd3LH5SSJqa3lv0jO8dOnoVhugzIicQUZZBn6+ORSU15BRVKnFCXqoITiWX87xggqrVdIA/u6OPDY3jh3Hi1i8VcuRWJuxFkCLD5hXOhapo7/vz+HNPw/z96+S2JvZ+ntw/eE8lu3O5tbJ0QR7OnXBFZ0al591Oc52zo1WBTPjAikor2H78SJtg5O3pgJc00pDlt6OhSEYGxnaYtpouzGvCFTP4q7BzjWQC8vK+d/gO7ln+D3szt3Ndcuvo6quuZTBeUOCcbDTNXxwbZraKvj1oUZfxIfbkToKcLjoMH/7/W94Sx1vF9fhETy8zZebGj4VO50deVJbte3JMAWM85O1Vow9jDXJmg98QiuZIfMSgxkV6c1/fz1IaVUtazLWEOEeQbh7uOYWgobU0bp6I//+eT+RPs54ORu47fMdLda11NUbeWzpPkK9nJpVb59pPBw8uGTgJfyS+gtppdrnxtz6cld6kTbIXF3clwPG1SUU6rQYwdio9lWXt4pbIEhjixL6ncEmDQEu2gfVsbKIa+Kv4cUpL1JYXcgvqb80G+rhZGBWfCA/JLWiDGgrrPg/WP8KfHF5Q9XkYVPqaL9WXEMZZRnc+OuNGHQG3imswi9sbLsaprjbuzMueBzb81dhp7PIHDLWQd6hrrmmLmRNci4hnk5E+bb8txBC8NCcQeSV1fDSyn1sydqirQZACxQ7+4Kblur51dY0DueWc//sQbx02VCO5Zfz4Pe7rbopP998nIMnSnnoL4NwNJz+msy2uCr2KvRCz4d7PgTAx9WBQHfHk/0lbEGBtKqYQqFHGu2I9LFeTdwhRt8MD+WAYxecqwm2aQjMdyOmWoIRASOI9ozm8/2fW/2QXTIijNKqupY7LtkCB3+GzW/DgJlQnA5LbwcpOZJbhoeTAR9rwS0grzKPG3+9kcr6St4a/ShhhWkQOb7dLzsjcgYnKrKJDMnT3HP+cdqOHuYeqqs3sv5wPhNifFuXbMjaRcKSc1jq9wYndv6bGmMNEwJHa/uyd2puISEoq67jhRWHGBnpxbmxAYzp58Pfpw/gh6RMvtrSeHVaWF7Df389xLhoH2bEBZ7Gq2w//s7+zO0/lyUpS8ir1D5nccHu7M009Zdo0BvqwyuCqmIK0SGNjgR6dIFWlt7QoY5zHcE2DYHeoFU2mgyBEILLBl7G/oL97Mrb1Wz4mH4+hHk72a57qCQLltyiuSwu+RimPQz7foCt75syhlysfvmty1jHZT9dRm5lLq9Pe50BZp2UiHHtfukpYVOw19nj7LWHXenFSJ/+oLfvcZlDO9OLKa2qazE+0MCur6DgKLG6NAJdN+JkNDL8w4vgnWmQs7/BLfT2n4fJK6vhgdmDGv62t06JZly0D48s3cvB7JPugf+uOEhZdR2PnBfXZbpBXcG18ddSJ+v4eN/HAMSFeHA4t4zKmnoLBdK+bAhKKAJkvRNBHt0Xs2kPtmkIQFuCWxSVndf/PFwNrnxx4ItmQ3U6wcXDw1iXkn9SNM1WMBphyc2atvqF72tCaGPvgOjp8Mv96HP2NosPlNWU8ej6R7n5t5txMbjw/oz3SfRPhGNrNQPs31SKqmVc7V0ZFzKOPLmF4spq0otrNVXOHrYiWJOcixAwLroNLZ+U3yByPPq/J7HMsx8OFSEUxF6rGTcHN4iezomSKt5Zc5Q5CUEMDT/ZrEevE7x46VDcHA3c+vl2Kmrq2JdZwuebjnPVmAgGBLid5qvsGOHu4ZwbcS6LDy6mpKaEuGB3jBL2Z5f0/RhBfS3UllMsJXqccLLvfndda9iuIWgiM+FscGZe9DyWpy5vWMpacuHwUISAb7alN9vXp1n/MhxZBbOeBr8B2jadDs5/C6OTF4/XPMdAL4vhmes5f+n5fJ/yPdfGX8vi8xYT72sqBEtdB+FjO9x+cUbkDErr8tE7HTfFCeJPS1FZWkEFV7+/udHddntZk5xHQohH65WjRWmQewCip5NakkqJMZfKunHcmDUX4zXL4L5UiJrI878eos5o5J8zmstQ+7k58NJliRzOLeP/luzlsR/34uFk4K7pAzo85zPBdYOvo7y2nK8OfNWgurk3swScPAHRd2MEpoBuiazHQde+1OruxIYNgW8z4blLB15KnbGObw9922x4iKcT46N9+WZbulWZgD5JxjZY+QQMmgvDFjbe5+LLkYkvESmy+cvx/1BWXcpjGx7jphU34WTnxMezPuau4XfhoDdJKZdkajnyEWM7PI3JYZNx0Dtg77mb7ccLtYBxWXabelEdwWjUVEJXH8rlxd86FoguqdJkJdp0C6X8pj3GnMOa9DUA3DLmL+zOKOZ7U7/aA9klLN6WxsKzI1uU7BgX7cvtU6L5dns6m44WcO+MgXg4d73+TFdwlvdZjA8Zz6f7P8XbFTydDezLLAadXjMGfdUQVBUBUE49znbKEPRcXHyb6Q1FeUQxNngsiw8tptZY2+yQS0aEkVFUyTpbaGxfXQrfXq81xJj7crMglZSS3+vduMl1HP+u2sA5iyfzXfJ3/DXuryyes5ghfkMany91nfYY2f74gBkXgwsTQibg6LGHTUfzLCqMu8499Pnm46w/nM9ZgW78sjebY/ntz2/fcDifeqNsNW0U0AyBRxj4DmBtxlqiPKK4ZuRwhoR68OzyA1TU1PHvZQdwc7DjtqmtVwXfOX0Akwb4MSLCi8tGhrd7rt3B9YOvp6CqgCUpS4gLdmdPhkXAuK/GCExZdZXU4mbfs1x21rBhQ+CnvQmbtH67bOBl5FTk8MfxP5odck5sAB5OBhZvtQH30LJ/QmEqXPC25tc38fux31m0ZhFTv57Kq4duZpP/cQ47uzOzpJhPxjzJ3SPu1qQSmnJsrdYQ6BTlE2ZEzqBOFHOgcCdlngO1jV1kCNIKKvj3sv2Mj/blo2tHYacTvLf2aLuPX5Oci7NJpLBF6mo0F1v0dCrqKtl6YisTQiag0wn+b04sJ0qqufnT7fx5KJfbp8a0KU6m1wk+/OtIvrrp7FaL8noCwwOGM9R/KB/u/ZBBQS4czC6ltt7YtxVITb0IanW1eDp0fbpnV2PbhgDZ7I5kYuhEgl2CrQaNHQ165icGs3xvNkUVNc329xmSPoedn8OEexvdwa/LWMffV/2dDZkbGBk4khjdX/HKf5RlF/7MIxWQsPwxqCyyfs7UdRA+RnMJnAITQyfiqHfGzmsdW/IM2v8vp/OGQJoaxwghePrCwQS4OzI/MYTFW9Oa99ptgbXJeZzdzwd7u1Y+TmmboKYMoqezOXsztcZaTVYCTar5LwlBrD6US6iXE1ePjWjX6woherwRMHNd/HVklWeByx5q6o2k5JT1bQXSqmJqAamrx8e5k9ISZwDbNQQNtQSN3UN6nZ5Lz7qUrSe2WpWduHhEGDV1Rpbu7EVCdL89Cp9domX+tEXGNvjx7xA5ASbd17C5zljHc1ufI8wtjBUXreDZic9SnjeCGO9wreLxko+1GMC31zVvsF2Wo1UDdyBttCnOBmeuGnQ1Bve9/HxoS5dJTXy2SXMJPTB7EKFemk/+hon9qKo18unGY20efzy/gtT8ina4hVaAzgD9JrEmfQ1Odk4MDzhZXb1o5ln093Ph0fPi2qXi2tsYHzKeAOcAUipWAaaAcV9WIK0qptSUFOHv6tm9c2kHtmsIXEyBPSt9CS6IvgAHvQNfHmjeFjE+RGv43atqCo6ugeTlsPhqzUXREmU58NVVmqbJxR+C3q5h1/cp35NSlMLdw+/GXm+vic3ll9Pf35Q6GjEWZj+n+cF/e6TxeY+Z4gOdMAQA1w5eiE668GfOx1rmUM7+5kanA1i6hC4fdVICYECAG5MH+vHRhtQ2q8nXpGjvnwkD2goU/w7hY5D2rqzNWMuYoDHY60+6f8K8nfn9nslMj7Uuwtfb0ev0nNf/PJLyNuHkWK5VGPdlQ1Bd0mAIgt1acRn2EJQhsNKy0tPRk1lRs/jxyI+U1JQ023/JiDD2ZJScLJfv6VTkaUHf5F/h+5usf3nW18LihdoH87JPG2Q4QKsLeHXHqwzzH8a08GkAZBRWUlNnpJ+lnMKIv8LIGzQZCsvewqnrwOACwYmdugxXe1cS3S6kwm4f6x2doa4KCo60fWBTjEZk/hEWfacVDz594eBmhVg3TuxHXlkNS3a03B7DaJT8b1cWwR6Ojf8OTSnJ1ArgoqdzpPgImeWZDW4hW2Ju/7nUy3oCg/exL7NEixHUVUJNH6zNqSqmzGQIQjyUIei5mL/oWkhBvPysy6msq2RpStPWCjA/MQQ3Bzte/t26YmmPo6IA4ubD9Mdg73fwv7ubKxj+cj8cXw9zX4Ggxhk/7+15j4KqAv458p8NX5iH80xic/5NxOZm/ltzKy29A9K3aduOrYOwUVpFdye5IvYyjLXu/PfEZiScmnto81vIV0eQkpLMA3856RKy5Ox+PsSHuPPOmiNW04WllDy8dA/rD+dz/YR+rVf0pvyuPcacw9bsrdr5g8/u+Lx7OVEeUST4JlDjtJl9WcUYnfpwUVlVCXl22vvK19mze+fSDmzXEDh5gdC12LIy1ieWIX5D+PLglxhl405mHs4GbpjYj1/3nWBnWtEZmGwnqKuG6hKtknr832H83bDtQy1uYGbHp7DlHTj7Nki4uNHhmWWZfLz3Y87rdx5xvnEN2w/naIag2Z2w3gAXf6TFDb68Qvuiztl3Smmj1hgbFUht/jQOVR5hjbNzxw2BlNRuehedrOfikHyuGGU99VIIwQ0T+nE4t5yVB3Ka7f/vr4f4dONxbprUj2vHR7X+mikrwC0Y/GPZV7APTwdPQl3PXPvInsTc/nMpMaZRwXFyjab3Tl/MHKoqJl+nZc+5GtrZtKkbsV1DoNNrAeNWehdfOehKjpUc45ejzVVJ/zouEi9nA/9d0b7io6+2HOfndjYX6VLMHzKz2uO0h2HEdbDuRVjzPKRvhZ/ugqhJ2oqhCS9ufxEhBHcMu6PR9iN55Xg6G6x3UnLxgcu/1LJkPvyLti2ia1whbo4GBjhPw2D05RVfP4zZHdQcOrYeQ2EKANcPqGz1Tn724CBCPJ14e01j99M7q4/w6h8pXD4qjEUzm1f/NqK+Dg6vguhpIAR78/YS6xPbozSBziQzo2ZiJwwYPLdzuMz03umjhiBXpxVTutur9NGeTROZiaacG3kuA7wG8MqOV6itb1xg5uZo4G+T+7P6UC6bj7a+tN14JJ9F3+3mpS5yJeWXVbe/Y5r5Q2Z2hQmhBXUHXwy/PwafXKDdvTcJDgPsyt3Fz0d/ZmHcQgJdGqtaHs4po5+vdbE5QCv6uuBtqCwEO0cIGdaBK2ydMVF+VORM54Be8mthxwxB5cb3KJHOFBv88Cxp3Ygb9Dr+Oi6SzUcLSDKt/BZvSeOpZfv5y+AgnpzfPLbQjPQtUF0M0dOpqqsipSiFOJ+41o/pw3g4eDApbBIG9yR2F5n+dn0xYFxdQj6aK7RpP+6eiG0bAmefVg2BTui4c9idpJel813yd832XzUmEn83B55bfrDFL+biylruWbwTKSE5p6zTPQ0OnShl9L9+Z/nedvYuNV+fs0V6o04H89+AgbM1bf/LPj+5YjAhpeQ/W/6Dr5Mv18Vf1+y0R/LK22xGw1l/gb/8Fybco4nVdRGj+/lQVZhAhM6d1+xrqWuj93QDFQUYDv7ID8bx2IcNa5db6bJR4bg52vHOmiP8vDuLRd/tYkKMLy9cmti+HP6U30Dood9kDhUeol7WE+vTftG9vsj50fMRduWsLDAZ4j5oCIyVReQJPSBwMSiJiZ6Ni1+rriHQWggO8x/Gm7vepKK2cXaDk72e26ZGszm1oKE7VVMe+WEP2SVV3DypP/VGyYFTEDSz5OMNqdQZJRuPtHM5bVoR1Do2/qJHb4BLP4O79zVIH1vy67FfScpN4vahtzdrLl9SVUtuaXX7GtaPvB4m/bN9c20nIyO9EELHBLuJpNob+HHX++06rnzLZ9jJGk7EXIZT6BCttqG2eVc6S1wd7FgwOoKfd2dx55dJDA334q2rhrdePGZJygotUO7kyd58zfDY8ooAYGzIWOxx50CtFjjvi8FiY2URJTo9DjpndKLnf832/BmeTlz8rKaPWiKE4K7hd5FXmcdn+z9rtv+ykeGEeDrx3K/NVwVLd2ayJCmTO6bGcOUYLSjZmZTT0qpavt+upTPuaG+Q2rQimPHWHj5Yd5SaOovAt05nUoFsTHV9NS9se4EBXgOY139es/1H2tGV7HTi6WzPwAA3MorHEF9dzRuHv6Wmvo0qYCmp3PgeScb+zJ95rlaQJo2aGmgbXDM2Ejudjn5+Lry/cCTO9nZtHgNodRlZOzXJbmBf/j68Hb2budlsDYPOwGCPKdQ77afA0bOPxgi0OgKXXhAoBps3BL6aOFRrRVZAon8ik8Mm8/6e9ykyqQqasbfTcef0GHalF/PrvpPumsyiSh76fjdDwz25dUp/Qjyd8HQ2tNmAvDWW7MigvKaeMf282Z9ZQnVd224mWZ6HEUG+0ZXHftzHuS/8yc+7s1qNMby24zUyyjK4d8S96K1IQhxpZ5/i08noKG9WZNhzR2kNWbWlVo20JeWH1+NbeZTdgecT7e+mFaRBu9xDgR6OLLtzAl/ffHbHVD7NaaMmQ7A3fy+DfAbZbKDYkrnRcxGinh9d3fuea8hoRF9TSrlO4N4LBOdAGQLtsY1VAcAdQ++gvLac9/c0d0NcMDSEfr4uPP/rIYxGidEouWfxTuqMkhcuScROr0MIQXywx0nlxQ4ipeTTjccZHOLB1WdHUlNv5EBW226m4vxsCqUrD50Xzwd/HYm9nY6/fbadi97cwLZjzX3r6zLW8cHeD7hkwCUt5rofzi1DrxOEe1uXST4TjO7nQ2WtkXinfkyUDjy/7Xne3f1uiwYubcUblEonhs0yxTu8o8DOqd3pp9H+rrg5drAOImUFuPhDYAKVdZUcKTpi824hMzNihlFfFcRSR13HVwSZO04a2Z5ITRkCIxU6iddp6C98OrBxQ2CWmWjbEMR4xXBe//P4/MDnZJc37l1sp9fx93MGcPBEKT/uyuS9tUfZcCSfR86LJdIizz4uxJ2D2aWN3TPtZEtqIQdPlHLlmHASwzwBGjJZWqMkP5sC6c6ISG+mDPRn2R0TePqCwRwvqODCN9az6NuTrTnzKvN4YO0DRHtG84+R/2jxnEdyy4nwdm6/n/w0MCpKi3kcs+vH81k5zIqcyUvbX+LRDY82kxCvKi0g8sRytrhNJS4qWNuo04P/WV0iXGcVYz0cXqmtBnQ6DhYcVIFiC1wc7HCvG8Mhg5HDVa3H6Rqor4U//gXvTIUvF2ipuT2Rau1mr0YPHr1AeRQ6aQiEEN5CiBVCiGTTo9VaaiHEQtOYZCHEQtM2ZyHE/4QQB4QQe4UQT3dmLqeEOZOmpYBxyu/w+tgGSdlbEm+hXtbz5s43mw2dMziIswLdePrnA/xn+UHOjQ3gkhFhjcbEB3tQU28kOafjAeNPNh7DzdGOuUNCCPJwxM/NoV3FbLWlOZTq3Ik0NTmx0+u4bFQ4q+6dzMXDQ/lySxrZxVUYpZEH1z5IeW05z0581rqUtAlzn+LuxNfVgf5+LmyrCsKhupinE27jxoQb+S75O/72298aSYPsXPY2jtTgN+mmxifpIuE6q2Rs11JnozVJjn35Wkc1tSI4yRCvqegk/GBsh7s0LwXeOxf+fEZrdVpXqQX7eyKmXgR1dvW9ohcBdH5FsAj4XUoZA/xuet4IIYQ38AgwGhgFPGJhMJ6TUp4FDAXGCSFmdXI+HaOtFcGW97Q7RlNnqRDXEC4deCnfp3zPkeLGRUY6neDucwaQVVyFu5OBf1/QPMc8PkSTo93bQfdQbmk1v+zJ4uLhYTjZ6xFCMCTUk6T0ojaP1VcVIJ19m83FxcGOK0ZrAeyktEI+3vsx6zPX88+R/yTGK6bF89UbJal5Fd0aHzAzup8PK/K1/6EuZz+3D72dJ8c9ybYT27hq2VWkl6ZTW1eP94EvOGwXQ/yIiY1PEBCv3QSUNa8cPiXqajSBvxWPaJpOQgf9pwJafMDb0ZsA574pKncqDA0JI6Lcjf/ZG6lvSTxQStj6Prw1QVO3vfgjuPBdbV/WLuvHdDcNvQjqekUNAXTeEMwDPjL9/hEw38qYGcAKKWWBlLIQWAHMlFJWSCn/AJBS1gDbgTNbd99ajKCq5GRrwYM/N2y+YfANOOodeXXHq80OOSc2gLvPGcDbVw/Hx7V53nyEtzOuDnbs6WDAePHWNGrrJQvGnJRDGBruyZHccoorm3dSM5NXVo1rfTGOnv5W9w8KcsegF/x+ZBsvbX+J6eHTuXjAxVbHmkkvrKCm3tjtKwLQAsZJ1UHakxNaYdm86Hm8Nf0tcitzWbBsAR//+g4xMpW6xKuaB2kD4hode0qU5cKWd+GLy+HZKPhoDmx4DdyD4YJ3Guoz9uXvI84nTgWKLYgLdiesJIgcvZ5fDjfX9KIsB764TKt8Dx8Df9ugaWb5xGhFitk91RAUI4FaUWMzK4IAKaVZNyEbsHa7EwJYajanm7Y1IITwBM5DW1VYRQhxoxBiqxBia25uO32KbeHooWnEW3MNHVoO9dXgO1BT7TT5I32cfFgYt5AVx1awI2dH0zlyx7QYhrXQqUqnE8QGu3cohbTeKPls4zHGRfs0ugsfEuoJwK5WVgXbU/PxohQv3yCr+x0Nes4KMrCy4Hl8nX15dOyjbX5RmVNHe8SKIMqHMpwpcQzRVm9rX4TSE4wKGsWnsz/F2c6Z10+8TpK9KwOmX9P8BP5mQ9AJ99C318L/7tHOkXApXPYF3HcUrvkJBl8EQEVtBUeKj6j4QBPigj1wKQ8joraWResf5s6Vd55caR/fCG+O17q6zXoWFnwL7qb3sd5Ocw9l7ey2ubdGTXkhFUIgkbgZ+oghEEL8JoTYY+WnUYK51NI1OtzVXQhhB3wBvCylbFFTWEr5tpRyhJRyhJ9fG9rv7X9xq72LAdi3BNyCYMoDWiPqtI0NuxbGLSTENYT719xPaU3H/P3xwR7syyqh3oqipTVWHsghs7iKq8Y07lo1OFRzM7UWJ9h35Bh6IfELCLa6X0pJvfe3VJPHv8c/jYdD252UDptSR9tVTHaaCfRwJMLHmbe97gLPcK0PwvOD4Isr6Jd9iMv87sfLWMedwQFk11mROnbx0f7HnTEEeckw+BK4cyfMeR7Omg0OjT/8BwsPYpRGFR9ogreLPXpHPxZnZHN79CVsyt7EBT9cwBNLryDv47lg7wI3/AGjb9JqXiwJStBWBO2VWjmDlBbnN/Qi6DMrAinldCllvJWfH4ATQoggANOjNWdrBmAZNQ01bTPzNpAspXzxlK+iM7j4No8RVJdC8gqInadlfejtG7mHXAwuPDPxGbLLs3li4xPt1/0B4kPcqao1NuTit8UnG48R4O7A9EGNF1seTgb6+7mQlNby6uLIseMAGNysu4aWH1tOWu06avKm4caAds3ncG45Xi2JzXUDoyK9+fREBMa//gK3boGzb6Xu+Cb44lIu+v0i3sjOocZgxy2/32K1twT+safuGqqtgtIs8InWbipaoCFQ7KsMQVO8fANxlpIbgybwv/O+4xJDAN8V7GJ2WCBvjL2aCu9I6wcGJmhB2aLjZ3S+7aG8uKDBENhKjGApsND0+0LgBytjlgPnCiG8TEHic03bEEI8CXgAf+/kPE4dZyuGwOwWip0PDq4QNREO/K/R3ccQvyHckngLPx/9maXW/JstYA4YtydOkJpXzupDuVw+Khw7ffN/1ZAwT5LSiqwaouq6evJOmOyti4/V8y8+uJhg5zBq8qa2W05byxjqOW/u0f18KKqo5VBOKTmO4TxQfglxxS9wh7yXHJ9R9IuezYtTXia1JJW///H35hXIAXGQe1BLTewo5i8hr9Z7DO/N24uvky/+ztYNsi0TGKStVmsyduHz5ZU8cHAjS4LmMD58Oq/vfZ+FvyykrMbKTZO5Z0b27jM42/ZRXVZAvtBqTvrMiqANngbOEUIkA9NNzxFCjBBCvAsgpSwAngC2mH4el1IWCCFCgQeBWGC7ECJJCHF9J+fTcazpDe39XnMZhI3Wng+cpWUs5DVWq7wu/jpGBIzgqU1Pcayk7f62oOn3Oxp07Sos+2LTEfQ6weUtaOYnhnmSV1ZNZnFzvZy9mSW4mdPyLAXnTJwoP8HW7K3Mi56Du6N9uyUrjuSW078HBIrNjDbVEzy2dB+T/7OKxVvSuHxMfx75xz+JuuMn9Jd+zOjgMTw+9nG2ZG/h4fUPNzacAfFQXwP5hzv+4kWm/7lXZKvDzIFiRXMiwrT3tv3v/6cZ5Es/JWLG0zw/5XlemfoKyYXJ3LXqrmbqv/jHallZPTBgXFNeRI5OS9fuMzGC1pBS5kspp0kpY0wupALT9q1Syustxr0vpYw2/Xxg2pYupRRSykFSykTTz7udu5xToKkUdXWZli00aO5Jv+QAU1brwWWNDtXr9Px7wr8x6Azct/q+5m9WK9jpdQwKajtgXJP8B3/fMo0FMXUEuFvP6TcXllm7m9+WWoiPMMUvnJuvCJanLkcimdVvFkPCPNu1IigoryGvrJ1ic2eIUC8nQjyd2HAkn0kD/Fhx9yQenRvXLGvrvP7nccfQO/jfkf/xyo5XTu7oTOZQYar26NnyikAFilsnJiKMammg2DkCblgJg85r2Dc5bDKPjn2UjVkb+b/1/9e4QZS9M/gO6JEppLKymHy99v6zlRVB78fFB2rLT/ZNPfSL1gs3bv7JMR4h2lL0YPMGNYEugTw+9nH25u/llaRXmu23RnywB/syS6y2QDRzcNsqnKjmRrf1LY45K9Ade73OuiE4VkiUs2ml4NJ8RfBL6i8M8h5ElEcUiWGeHDxRSmVN69pFqw9pKyfzXXhPQAjB21cP54dbx/HGlcOJaqV38PWDr+fCmAt5Z/c7fLb/M0prSpE+MaCzO7WAcWGqlsbo2nJtwIGCA0ikWhG0QJC3Gzc5P8c9ni+B38Bm++dHz28w4C9tf6nxzsCEHrkioLqEYlNBpq3ECHo/TZvY71uiNXoPG9N43MDZkLbJavHZtIhpXDzgYj7Y8wEbMje0+ZLxIe6UVtdxvKDlpt1ZqZoqZsjxpdabzaMJ3sUGuzdz60gp2XqskAGu1WDv1qwXQFpJGrvzdjMzaiagrSzqjZLdbaxSVuw/ga+rQ0Pqak8hLtiDIabVUWsIIXhozENMCJnA05ufZuwXYxn+5dlMDwvh0oyf+Ntvf+PhdQ+z4tiKttVMQXMNeYY3z2ixwCw9rVYE1hFCkDD0bFYereBEiXVJ8OsHX8+lAy/l/T3vNxYXDBwMJRlQ3rPUSw21JZTaackUakXQW2ioLs7V3ELJKyB2bvMP94CZgNQCyVb4x8h/0M+jHw+sfYCCqtbVFOOCWw8Y780sxqk8HaPQI0oytFzqFkgM82R3ejF19SeXzWkFleSVVRPuWGE1UPxLqraymRWpubyGtOJiMlNTZ2T1wVymneWPrj0NWXoodjo7np/8PM9OfJZ7R9zLlbFXMsbeF5+aSgqrCvkj7Q/uXnU3U7+eypMbn2RX7q6Ws8IKU1t1C4EWH/B38sfPuYtSnvsg84aGYJTw485Mq/uFENw/6n6mhk3lmc3P8Gvqr9qOoATtMbtn1RM41pVRYTBgr7PHQd91DZlOJ8oQNOgN5UPycs0tFDu/+bigIVoD8iZxAjNOdk48O/FZSqpLeGLDE62+5IAANwx60WLA+PNNx4nQ5VAXPRMcPSHp8xbPlRjmSWVtPSkW6ajbjmuGyE9fajU+sOzoMob6DyXIVSvQ8XV1INTLqVURu81HCyitrmN6bO+XSHC0c2RW1CwWxi3k7uF382TEebyekcaX097gj0v+4M3pbzI2eCxLUpawYNkC5i6Zyzu73qGwqolaa+HxNgPFe/P3EuurVgOt0d/PlYRQD5YkZbQ4Rq/T88zEZxjiN4T719zP1uytmmsIelScoLquHmdZToXBrtesBkAZgpP+8/JcLVvINUArZ2+KEFr20OGVLXa1Gug9kFsSb+G347+xPLXJyqGuuiFF0d5Ox8BAN6u9Ccqq6/hxx3FCRD72gWdpvYUP/ASVRVZf03w3n3T85P6tqYW4OtjhUlfcLGMouTCZlKIUZkbObHae1gzBb/tP4GCnY3x083hDr6ehN8E+7HR2jAsZx7MTn+WPS/7gsbGP4e3ozcs7XmbeknksO7JMWyFUFmq9iFtJHS2vLSe1OFW5hdrB/MQQ9mSUkNKKIKOjnSOvTH2FYNdg7l51N8V6PXiE9ag4wYniatypoMpOrwxBr8LsGio6drKIzEozFkCLE9RWQOqaFk+3MG4hcT5x/GvTv066iEqz4Y2x8N0NDeO03gTFzdwOPyRl4F6bi556ze2QeIW2StnbvGcyQKSPMx5OBnZaSE1sO1bI0HBPREV+s0Dxz0d/Rid0nBt5bqPtQ8M8ySiqJLe0utlrSCn5bf8Jxkf74mTfwt+mN2POHMrZ12izm70bF8RcwEezPuLbud8S6hbKfWvu446Vd3Aia7s2qBXX0P78/SpQ3E7mDAlCJ2DJDuvuITOejp48N+k5imuKNb2vwIQetSLIKijCQdRSpe898QFQhkArY7dzgqQvWnYLmYmaAPauLbqHQPNBPz7ucUpqSnh689NacPnjeZCfojXUMBEX4kFhRW2jGgBz85nxPpqeD16REDxUy5luwT0khDDdzWuri9KqWg6eKGV4uKf22hauISklPx/9mdGBo/F1amwgWosTHDxRSnphZZ9wC1nFLQicvFpNIR3gNYBPZn3CvSPuZWPWRuavv49v3FyQrRgCFShuP/5ujoyP8WNJUkablfoDvQdyyYBLWHxoMQd9wrTPVk35GZpp6xTkaZl1VXqJay9pUwnKEJzUGyo+3rJbyIydA/SfoslNtPJmHeA1gBsTbuTnoz+z8rM5WlCx/1StErVOu+OOD9YaVljWE+xIK2J/VgnzI00ZK14R2vwSr4D0LVrBjRUSQz04dKKUipo6dhwvQkoYFWKvVUdbGIK9+XtJL0tnVlRzte/4YA/0OmHVPfSbqQXntLP6aGWsEJp7qI0UUr1Oz8K4hXw791sG2XvzmK8PN+x8gbSSNKvj9+XvI8A5oJnRVVhnfmIw6YWVVjvnNeW2obfhbu/Ov8r2IZGnr69EBykq1LIKK0Tv6UUAyhBomN0ng+a27BYyM3C2pi+TldTqsOsHXMZAaccT+hKKL3gbhlyuNUsvOKq9VJA7ep1gr4Uh+GzjcVzs9QxzLwGhB3eTKnfCpdrzFlYFQ0zpn3sySth2rBCdgATv+sbXhhYkNugMTIuY1uwcTvZ6Bga4NXIxmfltfw5DQj3wb6GwrU8QEAcn9oGx7e5x4e7hvOsUy8PFVewpPMD8H+bz8vaXqahtnA68L3+fWg10gBlxgTgZ9Hy/o+WgsRkPBw/uGHYH20tT+cXFuccokZYUaams5cZaZQh6HeY4gWURWUvEnKuVtluI0DWjpgLDV1fxRFYmhXYGni3YrAmTgbaMRZOAjvZzZU+mljlUVFHDT7symT80BPuS4+ARqsntArj6a6+780ur7fks3TrbjxcyMNAd17rG8hL1xnqWH13O+JDxuNtbb5+XGK4FjC0L3XJKq0hKK2ometfnCIjTCguLUts1XFd0nIsdgvlh3g/MiJzBO7vfYc73c1h6eClGaaSspozUklQVH+gALg52nBMbwP92Z7WrnesF0RcwyHsQz/l4U2GO2XQzFSVaXLC0vkoZgl6HV5SWfRBuvVl7I1x8NQ2ilgxBXTV8dSWkrmXQnNe4dvB1LD28lDU1JmFWkyEArYexuYjr2+0ZVNcZWTA6QgtcN01LHLoAyrLhyB/NXtKc/rn9eCE7jhcxPMLzZIGcaUWwPWc7OZU5Vt1CZhLDPCmtquNI3kl/68r92rz7bHzATIPURDtdDIWp4BVBgEsA/5rwLz6d/SmBLoE8uPZBrlx2JV8f+hpQ8YGOcv7QEIoqavnzUNs9R/Q6PQ+MfoAcvY538raegdm1TXVZAbVAlbFGxQh6HdMf0XTP23ILmRk4S0tZW/4g/PIA/O9e+PFOWHILfDAbDv8Oc1+BwRdx85Cb6e/Rn0e3/odSV/9GfVbjgz3ILa3mREkVn206xrBwT2KD3Ru+ZBoRM0Pz9+/41OqUhoR58vv+HMqq6xge4XWyAtrUIevnoz/jZOfEpNBJLV6WNe2i3/bnEOLpxFmBvefu5pTwGwSI9hkCoxGK0xoZ6yF+Q/h09qf8a/y/yC7P5vltzwPKEHSU8TG+eLvYt1pTYEmifyLnOYXykb6S40WnIBzYxdRVFPW6XgSgDIGGvQu4dqDyM3ae1t1s8zuw/WPY8422Qji6Wmtic95LMOwq7dR6e54Y9wR5lXk84euN0WJFYJakfnfNEY7klmurgZpyraah6YrAzl5rgHJwGVQ0r1weGuZJjam6eESEN1SYyu6dfak11rLi2Aomh03G2eDc4mX193PF1cGuIWBcWVPP2pRcpg/y7/stFu2dwbtf+wxBaZamWNokY0gndJzX/zx+Ov8nbkq4icsGXoaPk3UJcIV1DHod5yUE8du+E5RWtU8a/K7+F2MvJc+sf+w0z651quvq0VWXUNYLDYFdd0+gV+IVCYva3xBjsN9gbku8jZd3vIxLTQkPS4kQWttKgA/WpeLhZOAvCUFQYMoMspaWmHgFbHoD9nwLo25otMscJ/Bz09xEVORpDXUc3NiYsZai6qIGSYmW0OsEg0M8GgLG61LyqKo1Mq2vxwfMBMS1zxCYVUdbqCp2Njhz29Dbumxatsb8oSF8tOEYv+zJ5uIRYW2O9wsfx81/Psx/dTtYnb6aiaETz8Asm3OiuBp3UUGxKbbXmwyBWhGcIa4ffD3Xew3hGyc7/r3+UaSUuDrY0c/XhTqj5KLhoTga9BYa91HNTxKUoAltJX3WbJc5/XNEhJd2916erwWKheCX1F9ws3djXMi4NueZGO7J/qwSqmrr+f3ACVwd7Bjdr+eojZ5WAuKh4EjbOent7EOgODUSwzyJ8HFut3sIn2gWVNQRqXfhmc3PUF3fvCjyTJBVXIkbFRTbawq4KkagaIYQgjtiLmNhcQlfpHzHf7b+ByklcSb30BWjTc1nGu42WyhUSlygFaadaFwF62Sv51/nx3PrFFN2UsXJYrIt2Vs4O+hs7PVtt5ccEupJbb1kb2Yxv+3PYdIAPxzs+mA1sTUC4gAJOQdaH1d4DBBaZpeiyxFCMD8xhPWH81tUJG2ETo8hII77a505Xnqct3e9ffonaYWs4ircRQVlDpohUCsChVWE7wDuKSjiCt+RfLLvE17a/hI3jI/iifnx9Dc3eyk8plUvWxGLA7T4BGjxiCZcOjK8Ie5ART64+JBdnk1WeRZD/Ye2a45Dwz0B+GTDMXJLq5k2qI8WkVnDrGaZsa31cYWp4B7STN5b0XXMHxqClPDh+tR2pZISlMDY7BTm9p/L+7vf52CB9eLL00lWcRVuVFBu7wQoQ6BoCa9IhNCxyLEfFw+4mPf2vMfa/M+5aozF3b9Z2ril4KxbkBaobtI2sxnleeDsy85crdAm0T+xXVMMcHck0N2RH3ZmohMwZaANGQKPMO0L/njLzYAAU3pv6/LTis4R5evC+Ghf3lh1mOFPruCur5L4ZU92y82TAhOguph/Rl+Ku4M7D69/mDpj85qb00l2cSXe+krK7LXCS2UIFNaxswfPCERBCg+NeYj50fN5Y+cbvLPrnZNjrNUQWCKE1qKvLUNgEpxLyknCUe/IQO/m3Z9aIjHMEylhRKQ3Xi5tu5P6DEJAxFg4tqFVCZH29CFQdJ53F47g3atHMCMukD8O5nDzp9sY+sSv3PTJVtYfbtIgyrSa8yg4yoOjH2Rf/j4+2ffJGZ1vZnEVXrpKyuwMCAQuhp7T27stlCE40/jGQH4KOqHj0bMfZU6/Oby842WWpCzRvnys1RA0O0cbhqCuGqpLwNmHnbk7ifeNx6AztHuKiSb30HRbcguZCT9bK9wrPGp9f22Vlj6qAsWnHUeDnumxATx38RC2Pjidz68fzSUjwth+vIjrPtxKVa3F6sA/VpNhydrFORHnMDVsKq8lvcaxkmNnbL7ZphhBqd4OV4MrOtF7vl57z0z7Cj7RkH8YpESv0/P4uMcZEzSGx9Y/xsajv2oy1219yfgOgLITLfYoMNcZVDq6sz9/f7vdQmamneVPtL8rf0kI7tBxfYKIsdrjsRbcQ8UmgTnlGjqj2Ol1jI325fF58Txz4WAqa+sbi9MZnLTPRfYuhBA8OOZB7HX2PLr+0cZN708jWcWVuMhySnW6XtOr2IwyBGcan/7al32Jprtu0Bl4fvLzRHpEcveG/+Owwa5tt4PvAO3RojitESZ5ib1UUSfrSPRL7NAUYwLc+O3uSYR4OnXouD6B70BNkvpYC72n26ghUJx+Rkf5YNALVic3kaEIOtmbwN/Zn3tH3svWE1v55tA3p31O1XX1FJRV4WisoFT0rvgAKENw5mkiPgfam+b1aa/jgJ5bAvzJc/Zo/Rx+Jn9/C7LUZnmJpGqtujjBL6FTU7YpdDrNPdRSwNhsCFSMoNtwcbBjeIQXaw41iRMEJmhuvVJNNv386PMZHTia57c9T3Z59mmdU05JNa5o6rNlGHtVDQEoQ3Dm8YnRHpvczQe5BvGq/yQK9TpuS3qhmaRxIzwjtKrhluIEJnmJneVpRLpH4uXo1RUztx0ixmqFZaVWvjyKjoHeQetdoeg2JsT4sS+rhLwyi+KxsFHaY9pGQKtHeGTsI9Qb63li4xNtNrzpDJlFlbgL7TNbKutaVPjtqXTKEAghvIUQK4QQyaZHq984QoiFpjHJQoiFVvYvFUK03B6qL+EWBAZnLU7QhLiKMp4trWd/4UEWrVlEvbGFVDm9HXj3b9UQSCCpKLnD8QEFEN5KnMAczNepe6juZEKMpqq7LsViVRCUqHUbTF3XsCnMLYzbh97O6vTVzfuIdyEbjuTjgdkQ1NpcjGAR8LuUMgb43fS8EUIIb+ARYDQwCnjE0mAIIS4Ayjo5j96DTqd9iVuokDZQmMpk5zD+OfKf/JH2B89tfa7l8/jGtGwIyvM4ZmegqKakw/EBBZqv2eAMx63ECQqPKbdQDyAu2AMvZwOrLd1DdvbaquDYukZjFwxawCDvQfxny38or+36lpa19Ua+2Hyc8WFaqnVpfbXNxQjmAR+Zfv8ImG9lzAxghZSyQEpZCKwAZgIIIVyBu4EnOzmP3oVvtPVAr6lQacGgBVw56Eo+3f8p3yd/b/0cfgO1bmd1Nc33VeSR5K7pA6kVwSmgN0DoSOsB48I26jwUZwS9TjAu2pc1ybmNXT6R4zXhQAuFXr1Oz0NjHiKnMoc3kt7o8rn8vv8EJ0qqmR3jhATK6iptLkYQIKXMMv2eDVhznIYAlk1d003bAJ4A/gu04hDXEELcKITYKoTYmpvbdtOKHo1PtPaFYvklXl8LxekNXzL3jriX0UGjeWrTU+zP39/8HL4DQNZrvuymlOeR5OSMm70bUR5WxOsUbRMxTmtmb5miW1kI1cUqdbSHMCHGl5zSapJzLBwKEWMBCcc3Nhqb4JfAhTEX8un+T0kutLIa7wSfbDxGiKcTg30FFUJgRPa9GIEQ4jchxB4rP/Msx0nNLLc7GiOESAT6SylbuOVtjJTybSnlCCnlCD+/DvQO6In4RGtf4uYMFNCMgDQ2uB30Oj3PTHgGDwcP7lp1F8XVxY3PYU4hzbOSOVRRQJJBMMRvSK8qaulRRJwNSEjbfHJboak4SbmGegTjY7TvgdWW3cxCRmjB/CbuIYA7h92Jq70rT216qssCx4dzy1iXks8Vo8PRVZc0NKXpczECKeV0KWW8lZ8fgBNCiCAA02OOlVNkAJai4qGmbWcDI4QQqcBaYIAQYlXnLqeXYCWF1Fp+uo+TD89Pfp4TFSd4YO0DjQtjfE3ZR1biBMUVuRwWHa8fUFgQMgJ0hsZfKKqGoEcR4ulEfz8X1iRbxAkMjhA6wqoh8HL04s5hd7LtxDZ+OvJTl8zhs43HMegFl4wIg6riXtmdDDrvGloKmLOAFgI/WBmzHDhXCOFlChKfCyyXUr4hpQyWUkYC44FDUsrJnZxP78Cnv/ZoaQgaNO4b320O8RvCP0f+k9XpqxtrEtm7aCJpuc0Nwa5areJSxQc6gb0zBCc2Dhi38D9SdB8TYvzYdDS/sdxExFjI2glVJc3GXxhzIYN9B/Pfrf+ltKa0U69dUVPH19vSmBUfhJ+bA1SVnJSgNtiWIXgaOEcIkQxMNz1HCDFCCPEugJSyAC0WsMX087hpm+3i5KU1jWm6ItDZaeqXTbhs4GXMjprNa0mvsT7TIqXRWuaQ0UiSqEGPYLDv4NMzf1sh/GzI2A61ldrzwmPg6Kmpvyp6BBNifKmqNbLdUm4iYpzmZrV065nQCR0Pjn6QgqoCXk96vVOv/ePOTEqr6rjSrB5cVUxpL+xFAJ00BFLKfCnlNClljMmFVGDavlVKeb3FuPellNGmnw+snCdVShnfmbn0OnyaZA4VHtPu8HXNm8AIIXjk7Efo79mf+1bfR1aZKT7vOwDykrVm6maqitjpYM8AB99W+xMr2kHEODDWnuxPUJiq3EI9jDH9zHITFu6hsFHaTdWxtVaPifON45KBl/D5gc9PuW+BlJJPNh5jYIAbIyNN2fBVRZTaa5+5PhcjUJwmmqaQtvEl42xw5oXJL1BnrOPuVXdTU1+jGYLacijNbBhXV3qCXQ72JLq1fC5FOwkfDYiTaaSqD0GPw8XBjmHhXqyx1B2yd4HgoY0Ky5py+9Db8bD34KlNT52SKN3O9GL2ZJRw5dkRWmtYgOoSSg29rxcBKEPQffhEawqiZj9mO75kIj0ieXLck+zJ38O/N//bInPopHsoOXcnlTodid5nna6Z2w5OXpq88bF12qqr6LhaEfRAJsT4sjezidxExDjI3A411jPTPRw8uGv4XezI2cHSw0s7/JqfbDiGi72e84dauHKriikzaEVlyhAo2odl5lB1qaYP1I4vmWkR07h+8PV8c+gbvi415UNbBIyT8jT1xcSA4V09Y9sk4mxI36LJT9fXqNTRHsgEUxppI7mJiHFgrIP05nECM/Oi55Hol8jzW5+nqKqo3a9XWF7Dj7syOX9YCK4Odid3VJVQorfDXmePg753tTFVhqC7aDAEhzucn35b4m2MCx7Hv3a+QpKbd6MVQVJRMv51dQR1oCOZohUixkJNGRwwpRsq11CPIz7EA09nQ+M00vAxIHStuod0QsdDYx6ipKaEF7e/2O7X+3pbGjV1xpNBYjNVxZTp9b0uPgDKEHQf3v0Aoa0IOpifrtfpeWbiMwQ6B3K3jzu5efsa9iWVpzGkugbh4tvlU7ZJzAJ0SV9oj16qUrunYVVuwtFdk6VuqcGQiYHeA7ly0JV8m/wtSTlJbb6W0Sj5bNNxRkV6c1agRfWwlFrWkKDXVRWDMgTdh50DeIZrhqAhPz2y3Yd7OHjw4pQXKRNwT10atfW1nCg/QWZdOYm1UiusUXQe9yDt/3JiNyDAI7S7Z6SwwoRoX06UNJWbGKe59WqrWj32lsRbCHAO4ImNT7TZ8H5NSh7H8itYMCa88Y7aCpD1lEKv0xkCZQi6F59oTYW0MBUc3LXgZAcY6D2QxwMmscOg45kNj7MzdycAibre0zS7V2BeFbiHaAZc0eMYb5KlbuQeihwH9dVa0LgVnA3O3D/qfg4VHuKz/Z+1OvannZm4O9oxMz6w8Y4qTQKmlPpeFygGZQi6F98YU4wgVYsPmNPQOsDMqFn8taiErw4v4dWkV7FHMMhRuYW6lIiztUcVH+ixhHo508/PpXEaabjp/9ZKnMDM1PCpTAqdxGtJr7XYzUxKyfrD+Yzt74uDXZN6H5MhKOuFvQhAGYLuxSdaC0Smbzn1LxnfAdxZWMTZrpEcLT5KfL0Og3MvF+XraUSM0x5VxlCPZmKMHxuPWMhNOHuDf1yLhWWWCCG4f/T9SCl5ZvMzVsccy68go6iScdE+zXea0sBL62tUjEDRQcyaQ5WFp56f7hmBXm/Ps66xxHjFML2qFpytvFEVp453Pxj4Fxg4s7tnomiFc+MCqKo18tUWC9X7yHGa1ER9rfWDDv8BSZ/DvqWEZB/gpojZ/Hb8N1bvX9zsmHWHNbfT2GgrK27ziqC+qlfGCOzaHqI4bZhTSOHUDYHeDnyi8cw/xneXfwtPBYK1OxbFqSMEXP55d89C0QZn9/NhdJQ3r6xM4eIRoTjb22mruc1vQ2YShI08ObiuGn6+D7Y1VrxZCPwUEsi/1j3CyB3f4nTFVw371qfkE+juSD9fKzG4knRqgUpjjYoRKDqIeyjYmbJ7OuN2MIvP1ZRDXZUmaKdQ2BhCCP45cyB5ZdV8sC5V2xhh7j9t4R4qOg7vz9CMwLi/wx074OZ1cO2vGK78jgcH30KGwY63i3Y1HGI0SjYcyWdstM9JSQlL8pIblEdVjEDRMcz9i6Fz0gW+A6DwKJSaxOhUDYHCRhke4c20s/x588/DFFXUgKu/9vkw1xOk/AZvTdSSNC79DM55THP9BcZr2lLR0xg55k7musXwobOelGwt4+hAdikF5TWM69/CZyvvEKXekYCqI1CcCuY4gWd46+Naw3egSXZ3k/ZcxQgUNsy9MwZSVl3Hm3+a2rhGjNVaV656Gj69CNyC4MZVMGhOy+cYsABXo5HHNz6BURpZb4oPjLMWHwDNEJg+w70xRqAMQXcTOw/iL+xcAZi5W5m5iYpyDSlsmEFB7swdEsyH64+SU1IFEeOhugRW/RsSLoHrfzt5A9YCXkGJ3FtQxI7iFL5N/pZ1KXn083Mh0MPK57S2EorSKHXXagtUjEDRcQZfBBe937lzmA2BWS7ZRa0IFLbN3ecMoK5e8vLKZOg/FUJHwuzn4Py3NJnqtvCKZG5lLaPsfXlh2wtsPp7K2P4tfK7yDwOSMldtvzIEiu7B3Lay4LD2XK0IFDZOhI8Ll44M48vNaRyrctRWAaNuaH/Rpk6P8Inm/4weVNZWUe+1tNX4AECpkyegDIGiOzH3JtAZwKH3vREViq7mjmkx2OkFL6xo3te7XfgOIDL/GAmuF2Lw2InOpYVuZnnJgOi13clAGYK+g9kQuPieklSFQtHXCHB3ZOHYSH7YmcmB7OaN7NvEdwAUplKTMxq7+gBe2PE0lXWVzcflHQLPMEqNVQiEChYruhE/kyFQbiGFooG/TeqPq4Mdzy0/hd7Eflo2XnHaYSZ7/42Msgze3Plm83F5h8B3AGU1ZbgYXNCJ3ve12vtmrLBOw4pABYoVCjOezvbcNLEfv+3P4bvt6R072JSEESnTmR87kQtiLuDjvR9zqNDC1WQ0alLyvgMoqSnplfEBUIag7+Br6kimaggUikZcOz6KERFe3L14J/d/t+ukKF1b+GiGYIA+i5GRXtw9/G7cHdx5bMNj1BtN5yjN1HoR+ERTVlPWK+MDoAxB38HFVytKM715FQqFhrO9HV/cOIabJ/Xni81pzH9tHSmWDWxawt6ZEzp/Rrjk4mxvh4eDB/8Y+Q925e7im0PfaGPMbWJ9B1BaW4qbQa0IFN2JEHDzWphwT3fPRKHocRj0OhbNOosP/jqSnNJq5r66tk1XUVFFDfvrghigz2rY9peovzAmaAwvbX+J3IpcU8YQDTECm3QNCSG8hRArhBDJpkerLbaEEAtNY5KFEAstttsLId4WQhwSQhwQQlzYmfnYPI4eYGff3bNQKHosUwb6s+yOCcQHe3D34p384+udVNRYb0+58Ug+h43B+FQd02IBaMJ2D415iOr6ap7d8qy2InDwAFd/m44RLAJ+l1LGAL+bnjdCCOENPAKMBkYBj1gYjAeBHCnlACAW+LOT81EoFIpWCfRw5PMbRnPblGi+2Z7O3FfXsS+zeXrpupR8jutC0ddXQcnJ1UOEewQ3JNzAL6m/sDYvSQsqC0FZbVmvTB2FzhuCecBHpt8/AuZbGTMDWCGlLJBSFgIrAHOHj2uBfwNIKY1SyjwrxysUCkWXYqfXce+MgXxy7WiKK2uZ/9o6Plh3FCllw5h1h/NwCDpLe5LXuCjt2vhrifKI4sn6E1T69EdKabuuISBASml2oGUDAVbGhAAWLYNIB0KEEJ6m508IIbYLIb4WQlg7HgAhxI1CiK1CiK25ubktDVMoFIp2Mz7Gl1/unMD4GF8e+3Ef1320lfyyarKLqziSW05YTKI2MLexIbDX2/N/w+8lQy94S19GZV0l9bJ3Nq6HdhgCIcRvQog9Vn7mWY6TmimVLZzGGnZAKLBeSjkM2AA819JgKeXbUsoRUsoRfn6qJ69CoegafFwdeG/hCB49L5a1KXnMemmNJlYHDBsUDU5ezVYEACN1rswvLeOj4r1sz9H6FvRWQ9Bmq0op5fSW9gkhTgghgqSUWUKIICDHyrAMYLLF81BgFZAPVADfmbZ/DVzXvmkrFApF1yGE4JpxUYzu58PtX+zg803H8XI2MCjIQ6vRsWIIyEvmnoIi/vQJ5rENjwG9U2cIOu8aWorW5hPT4w9WxiwHzhVCeJmCxOcCy00riB85aSSmAfs6OR+FQqE4ZQYFufPjbeO5cWI/7pgWg04nTraCbUreITyl4J7hd5Ndng2Au6H3dSeDzhuCp4FzhBDJwHTTc4QQI4QQ7wJIKQuAJ4Atpp/HTdsA7gMeFULsAq4CVBK8QqHoVpzs9TwwexB/HRelbfAbCOW5UFHQeGDeIfCOYm7MBYwMHAn03hVBm66h1pBS5qPdyTfdvhW43uL5+0Cz7itSymPAxM7MQaFQKE4rZh2vvGStr7EZk8aQEILHxj7G27veJsard1b2q8pihUKhaI0GQ2ChYGqsNxkC7Ys/zC2MJ8Y9gZOdUzdMsPMoQ6BQKBSt4RkOeofGcYKiY1Bf02e0vZQhUCgUitbQ6bU7f8taAguNob6AMgQKhULRFk0zhxpUR9WKQKFQKGwD34GaO6i2Snued0jrBujs3b3z6iKUIVAoFIq28I0BaYSCw9rzvOQ+4xYCZQgUCoWibfxMHQBzTZlDecl9xi0EyhAoFApF2/hEA0IzABUFUJGnVgQKhUJhUxictDTSvIMWGUNqRaBQKBS2he8ALUjcxzKGQBkChUKhaB9+AyEvBXIPgN4ePCO6e0ZdhjIECoVC0R58Y6CuEg7/ocUMdPrunlGXoQyBQqFQtAdfU+ZQzt4+5RYCZQgUCoWifVhmCfWhjCFQhkChUCjah4sPOPtovytDoFAoFDaK2QD4RHfvPLoYZQgUCoWivZgNQR+LEXSqQ5lCoVDYFCOuBZ/+4ODW3TPpUpQhUCgUivYSnKj99DGUa0ihUChsHGUIFAqFwsZRhkChUChsHGUIFAqFwsZRhkChUChsnE4ZAiGEtxBihRAi2fTo1cK4haYxyUKIhRbbLxdC7BZC7BJC/CKE8O3MfBQKhULRcTq7IlgE/C6ljAF+Nz1vhBDCG3gEGA2MAh4RQngJIeyAl4ApUsoEYBdwWyfno1AoFIoO0llDMA/4yPT7R8B8K2NmACuklAVSykJgBTATEKYfFyGEANyBzE7OR6FQKBQdpLMFZQFSyizT79lAgJUxIUCaxfN0IERKWSuE+BuwGygHkoFbW3ohIcSNwI2mp2VCiIOnOGdfIO8Uj+3NqOu2LdR12xbtvW6r3XTaNARCiN+AQCu7HrR8IqWUQgjZjomYz2sA/gYMBY4ArwD3A09aGy+lfBt4u73nb+V1t0opR3T2PL0Ndd22hbpu26Kz192mIZBSTm/lxU8IIYKklFlCiCAgx8qwDGCyxfNQYBWQaDr/YdO5FmMlxqBQKBSK00tnYwRLAXMW0ELgBytjlgPnmgLEXsC5pm0ZQKwQws807hxgfyfno1AoFIoO0tkYwdPAYiHEdcAx4BIAIcQI4GYp5fVSygIhxBPAFtMxj0spC0zjHgNWCyFqTcdf08n5tIdOu5d6Keq6bQt13bZFp65bSNlut75CoVAo+iCqslihUChsHGUIFAqFwsaxGUMghJgphDgohEgRQvTp7CQhxPtCiBwhxB6Lbe2SA+nNCCHChBB/CCH2CSH2CiHuNG3v09cuhHAUQmwWQuw0Xfdjpu1RQohNpvf8V0II++6e6+lACKEXQuwQQvxket7nr1sIkWqS50kSQmw1bTvl97lNGAIhhB54DZgFxAKXCyFiu3dWp5UP0aq3LWlTDqQPUAfcI6WMBcYAt5r+z3392quBqVLKIWhp2TOFEGOAZ4AXpJTRQCFwXfdN8bRyJ40zDm3luqdIKRMt6gdO+X1uE4YATeMoRUp5REpZA3yJJo/RJ5FSrgYKmmxujxxIr0ZKmSWl3G76vRTtyyGEPn7tUqPM9NRg+pHAVOAb0/Y+d90AQohQ4C/Au6bnAhu47hY45fe5rRgCqzIX3TSX7qI9ciB9BiFEJFrV+iZs4NpN7pEktKLOFcBhoEhKWWca0lff8y8C/wSMpuc+2MZ1S+BXIcQ2k/wOdOJ9rprX2yAdlQPpbQghXIFvgb9LKUu0m0SNvnrtUsp6IFEI4Ql8D5zVvTM6/Qgh5gA5UsptQojJ3TydM814KWWGEMIfWCGEOGC5s6Pvc1tZEWQAYRbPQ03bbIkTJhkQWpED6fWYNKy+BT6TUn5n2mwT1w4gpSwC/gDOBjxNcu/QN9/z44C5QohUNHfvVDRp+75+3UgpM0yPOWiGfxSdeJ/biiHYAsSYsgnsgcvQ5DFsifbIgfRqTP7h94D9UsrnLXb16WsXQviZVgIIIZw4KdfyB3CRaVifu24p5f1SylApZSTaZ3qllHIBffy6hRAuQgg38+9osj176MT73GYqi4UQs9H8iXrgfSnlU907o9OHEOILNKE/X+AEWmOgJcBiIByTHIhZ6qOvIIQYD6xBkzY3+4wfQIsT9NlrF0IkoAUH9Wg3d4ullI8LIfqh3Sl7AzuAK6WU1d0309OHyTV0r5RyTl+/btP1fW96agd8LqV8Sgjhwym+z23GECgUCoXCOrbiGlIoFApFCyhDoFAoFDaOMgQKhUJh4yhDoFAoFDaOMgQKhULRQxFCXGwSEjSaGn6155hwIcSvQoj9JgHGyLaOUYZA0aMRQtSbFBb3CCG+FkI494A5TRZCjO3kOYKFEN+0PbLRMdcIIV41/X6zEOLqzsxB0bMwva8+bLJ5D3ABsLoDp/oY+I+UchBaoVmbhWXKECh6OpUmhcV4oAa4uT0HWVSWng4mAx0yBE3nI6XMlFJe1NL4tpBSviml/PhUj1f0DqSU+6WUB5tuN2lL/UcIsUUIsUsIcZNpeyxgJ6VcYTq+TEpZ0dbrKEOg6E2sAaKFEOeZ9OZ3CCF+E0IEAAghHhVCfCKEWAd8IoSIFEKsEUJsN/2MNY2bLIT4UwjxgxDiiBDiaSHEAqFp+u8WQvQ3jfMTQnxr+rBtEUKMMy2zbwbuMq1UJlgbZ20+lhdimtse0+/XCCG+E0L8IjQt+Wctxv1VCHFICLEZTVIBi3Pfa/o92vR32Gm6TvP8/2HxRWHuUeAihPifaeweIcSlp+U/pTjdXAcUSylHAiOBG4QQUcAAoMj0ftphMhb6tk6mROcUvQLTHfUs4BdgLTDGJKx1PZr65D2mobFoglyVJjfSOVLKKiFEDPAFYPazDgEGocl1HwHelVKOElozm9uBv6Pp1rwgpVwrhAgHlkspBwkh3gTKpJTPmeb2edNxpnM3mk8bl5iIppZaDRwUQryC1l/hMWA4UIwmnbDDyrGfAU9LKb8XQjgCOiHEuUAMmmtAAEuFEBMBPyBTSvkX09w92piX4jQjhNgEOACugLfQVGQB7pNSLm/hsHOBBCGEeVXpgfb/tgMmoL2XjgNfAdegSa+0iDIEip6Ok8UHYw3aG3og8JXQhLXsgaMW45dafOkagFeFEIlAPdrdkpktZsleIcRh4FfT9t3AFNPv04FYcVK91F1oyqZNaW3c0nYYAdAaihSb5rMPiECTCFklpcw1bf+qyTUgNM2ZECnl9wBSyirT9nPRvizMhsMV7YtiDfBfIcQzwE9SyjXtmJviNCKlHA0NMhnXSCmvacdhAri9qaEQWkOiJCnlEdPzJWhNmpQhUPRqKqWUiZYbTHfLz0spl5o+PI9a7C63+P0uNK2lIWhu0CqLfZbaM0aL50ZOfi50aCsPy+Ow+MKnHePKmw5uAcv51NP5z6YA/i2lfKvZDiGGAbOBJ4UQv0spH+/kaynOPMuBvwkhVkopa4UQA9BUVregqa/6mW4gpgJb2zqZihEoeiMenJQWXtjGuCwppRG4Ck2UrSP8iuYmAsC0sgAoBdzaMa6zbAImCSF8hCavfXHTAaZObOlCiPmm13YwucSWA9eaVyZCiBAhhL8QIhiokFJ+CvwHGNZFc1WcBoQQ5wsh0tFkxf8nhDCvAN4F9gHbTbGmt9CCxPXAvcDvQojdaDcE77T1OmpFoOiNPAp8LYQoBFYCUS2Mex34Vmhplr/Q/rtzM3cArwkhdqF9VlajBYp/BL4RQsxDMwAtjesUUsosIcSjwAagCEhqYehVwFtCiMeBWuBiKeWvQohBwAbTyqQMuBKIBv4jhDCaxv6ts/NUdA1SylXAqibbvuek0qjldiOasu4DVvatABI68tpKfVShUChsHOUaUigUChtHGQKFQqGwcZQhUCgUChtHGQKFQqGwcZQhUCgUChtHGQKFQqGwcZQhUCgUChvn/wHqt6bqXWGpygAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(np.arange(1000000,1000050,1), reverse_top10fft[1000000:1000050], label = \"FFT top-10%\")\n",
+ "plt.plot(np.arange(1000000,1000050,1), concr[1000000:1000050], label = \"Original Parameters\")\n",
+ "plt.plot(np.arange(1000000,1000050,1), reverse_top10wv[1000000:1000050], label = \"Sym2 top-10%\")\n",
+ "\n",
+ "plt.title('Parameter Values') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.legend()\n",
+ "plt.draw()\n",
+ "plt.savefig(\"ParametersWaveletHaar.png\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## FFT Training<a class=\"anchor\" id=\"ffttrain\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 85,
+ "metadata": {
+ "id": "e65Izyv0s-yE"
+ },
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "eqOXilqMs-yF",
+ "outputId": "06799a3b-983b-4f51-a7bd-a901c041bd05"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.136391 [ 0/735856]\n",
+ "loss: 1.387546 [64000/735856]\n",
+ "loss: 1.009362 [128000/735856]\n",
+ "loss: 0.568759 [192000/735856]\n",
+ "loss: 0.796950 [256000/735856]\n",
+ "loss: 0.670068 [320000/735856]\n",
+ "loss: 0.625332 [384000/735856]\n",
+ "loss: 0.557147 [448000/735856]\n",
+ "loss: 0.701893 [512000/735856]\n",
+ "loss: 0.670033 [576000/735856]\n",
+ "loss: 0.575888 [640000/735856]\n",
+ "loss: 0.654841 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 82.0%, Avg loss: 0.578153 \n",
+ "\n",
+ "loss: 0.776733 [ 0/735856]\n",
+ "loss: 0.519993 [64000/735856]\n",
+ "loss: 0.599282 [128000/735856]\n",
+ "loss: 0.885723 [192000/735856]\n",
+ "loss: 0.514714 [256000/735856]\n",
+ "loss: 0.539040 [320000/735856]\n",
+ "loss: 0.422559 [384000/735856]\n",
+ "loss: 0.382564 [448000/735856]\n",
+ "loss: 0.412677 [512000/735856]\n",
+ "loss: 0.360731 [576000/735856]\n",
+ "loss: 0.534333 [640000/735856]\n",
+ "loss: 0.379236 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.1%, Avg loss: 0.463718 \n",
+ "\n",
+ "loss: 0.367664 [ 0/735856]\n",
+ "loss: 0.339760 [64000/735856]\n",
+ "loss: 0.653718 [128000/735856]\n",
+ "loss: 0.410070 [192000/735856]\n",
+ "loss: 0.554535 [256000/735856]\n",
+ "loss: 0.578007 [320000/735856]\n",
+ "loss: 0.421670 [384000/735856]\n",
+ "loss: 0.599983 [448000/735856]\n",
+ "loss: 0.262858 [512000/735856]\n",
+ "loss: 0.333737 [576000/735856]\n",
+ "loss: 0.361296 [640000/735856]\n",
+ "loss: 0.468058 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.9%, Avg loss: 0.420345 \n",
+ "\n",
+ "loss: 0.443198 [ 0/735856]\n",
+ "loss: 0.507409 [64000/735856]\n",
+ "loss: 0.554008 [128000/735856]\n",
+ "loss: 0.304086 [192000/735856]\n",
+ "loss: 0.482780 [256000/735856]\n",
+ "loss: 0.349616 [320000/735856]\n",
+ "loss: 0.402055 [384000/735856]\n",
+ "loss: 0.345523 [448000/735856]\n",
+ "loss: 0.364194 [512000/735856]\n",
+ "loss: 0.310542 [576000/735856]\n",
+ "loss: 0.441185 [640000/735856]\n",
+ "loss: 0.276955 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.8%, Avg loss: 0.390410 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " topk = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat))\n",
+ " top10[topk.indices] = flat[topk.indices]\n",
+ " g.grad = top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# it converges slower than without gradient compression"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch.fft as fft"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.122382 [ 0/735856]\n",
+ "loss: 1.659356 [64000/735856]\n",
+ "loss: 1.175072 [128000/735856]\n",
+ "loss: 1.030752 [192000/735856]\n",
+ "loss: 0.891644 [256000/735856]\n",
+ "loss: 0.732518 [320000/735856]\n",
+ "loss: 0.613185 [384000/735856]\n",
+ "loss: 0.483264 [448000/735856]\n",
+ "loss: 0.580724 [512000/735856]\n",
+ "loss: 0.509457 [576000/735856]\n",
+ "loss: 0.661517 [640000/735856]\n",
+ "loss: 0.621521 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 81.7%, Avg loss: 0.570322 \n",
+ "\n",
+ "loss: 0.543810 [ 0/735856]\n",
+ "loss: 0.339085 [64000/735856]\n",
+ "loss: 0.495473 [128000/735856]\n",
+ "loss: 0.384833 [192000/735856]\n",
+ "loss: 0.418521 [256000/735856]\n",
+ "loss: 0.614597 [320000/735856]\n",
+ "loss: 0.515266 [384000/735856]\n",
+ "loss: 0.738823 [448000/735856]\n",
+ "loss: 0.423178 [512000/735856]\n",
+ "loss: 0.473593 [576000/735856]\n",
+ "loss: 0.518021 [640000/735856]\n",
+ "loss: 0.497685 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 84.0%, Avg loss: 0.474809 \n",
+ "\n",
+ "loss: 0.575689 [ 0/735856]\n",
+ "loss: 0.456497 [64000/735856]\n",
+ "loss: 0.429356 [128000/735856]\n",
+ "loss: 0.563055 [192000/735856]\n",
+ "loss: 0.486054 [256000/735856]\n",
+ "loss: 0.542747 [320000/735856]\n",
+ "loss: 0.441926 [384000/735856]\n",
+ "loss: 0.461542 [448000/735856]\n",
+ "loss: 0.502812 [512000/735856]\n",
+ "loss: 0.383888 [576000/735856]\n",
+ "loss: 0.266721 [640000/735856]\n",
+ "loss: 0.490470 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.8%, Avg loss: 0.423047 \n",
+ "\n",
+ "loss: 0.302303 [ 0/735856]\n",
+ "loss: 0.421864 [64000/735856]\n",
+ "loss: 0.376742 [128000/735856]\n",
+ "loss: 0.259237 [192000/735856]\n",
+ "loss: 0.368860 [256000/735856]\n",
+ "loss: 0.400204 [320000/735856]\n",
+ "loss: 0.310619 [384000/735856]\n",
+ "loss: 0.320007 [448000/735856]\n",
+ "loss: 0.305337 [512000/735856]\n",
+ "loss: 0.375540 [576000/735856]\n",
+ "loss: 0.362421 [640000/735856]\n",
+ "loss: 0.347816 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.3%, Avg loss: 0.400034 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " \n",
+ " flat_fft = fft.rfft(flat)\n",
+ " topk = torch.topk(flat_fft.abs(), round(0.1*len(flat_fft)), dim=0, sorted=False)\n",
+ " top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ " top10[topk.indices] = flat_fft[topk.indices]\n",
+ " reverse_top10 = fft.irfft(top10)\n",
+ " g.grad = reverse_top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.125680 [ 0/735856]\n",
+ "loss: 1.490153 [64000/735856]\n",
+ "loss: 0.797238 [128000/735856]\n",
+ "loss: 0.703639 [192000/735856]\n",
+ "loss: 0.862654 [256000/735856]\n",
+ "loss: 0.674491 [320000/735856]\n",
+ "loss: 0.633835 [384000/735856]\n",
+ "loss: 0.537149 [448000/735856]\n",
+ "loss: 0.579062 [512000/735856]\n",
+ "loss: 0.468447 [576000/735856]\n",
+ "loss: 0.488582 [640000/735856]\n",
+ "loss: 0.529873 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 83.8%, Avg loss: 0.489548 \n",
+ "\n",
+ "loss: 0.573154 [ 0/735856]\n",
+ "loss: 0.466781 [64000/735856]\n",
+ "loss: 0.468422 [128000/735856]\n",
+ "loss: 0.449423 [192000/735856]\n",
+ "loss: 0.357713 [256000/735856]\n",
+ "loss: 0.391187 [320000/735856]\n",
+ "loss: 0.500866 [384000/735856]\n",
+ "loss: 0.368405 [448000/735856]\n",
+ "loss: 0.423239 [512000/735856]\n",
+ "loss: 0.533780 [576000/735856]\n",
+ "loss: 0.623185 [640000/735856]\n",
+ "loss: 0.380635 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.9%, Avg loss: 0.411804 \n",
+ "\n",
+ "loss: 0.485906 [ 0/735856]\n",
+ "loss: 0.522850 [64000/735856]\n",
+ "loss: 0.474864 [128000/735856]\n",
+ "loss: 0.453226 [192000/735856]\n",
+ "loss: 0.311791 [256000/735856]\n",
+ "loss: 0.370382 [320000/735856]\n",
+ "loss: 0.415271 [384000/735856]\n",
+ "loss: 0.448348 [448000/735856]\n",
+ "loss: 0.416761 [512000/735856]\n",
+ "loss: 0.392923 [576000/735856]\n",
+ "loss: 0.408733 [640000/735856]\n",
+ "loss: 0.369844 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.8%, Avg loss: 0.382454 \n",
+ "\n",
+ "loss: 0.351067 [ 0/735856]\n",
+ "loss: 0.441320 [64000/735856]\n",
+ "loss: 0.376012 [128000/735856]\n",
+ "loss: 0.326137 [192000/735856]\n",
+ "loss: 0.326353 [256000/735856]\n",
+ "loss: 0.337223 [320000/735856]\n",
+ "loss: 0.377199 [384000/735856]\n",
+ "loss: 0.453688 [448000/735856]\n",
+ "loss: 0.394669 [512000/735856]\n",
+ "loss: 0.462621 [576000/735856]\n",
+ "loss: 0.365274 [640000/735856]\n",
+ "loss: 0.414022 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.0%, Avg loss: 0.381759 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " \n",
+ " flat_fft = fft.rfft(flat)\n",
+ " topk = torch.topk(flat_fft.abs(), round(0.2*len(flat_fft)), dim=0, sorted=False)\n",
+ " top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ " top10[topk.indices] = flat_fft[topk.indices]\n",
+ " reverse_top10 = fft.irfft(top10)\n",
+ " g.grad = reverse_top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 229,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 230,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.128546 [ 0/735856]\n",
+ "loss: 1.380654 [64000/735856]\n",
+ "loss: 1.055664 [128000/735856]\n",
+ "loss: 0.687121 [192000/735856]\n",
+ "loss: 0.728443 [256000/735856]\n",
+ "loss: 0.731651 [320000/735856]\n",
+ "loss: 0.649674 [384000/735856]\n",
+ "loss: 0.474646 [448000/735856]\n",
+ "loss: 0.653415 [512000/735856]\n",
+ "loss: 0.450781 [576000/735856]\n",
+ "loss: 0.629819 [640000/735856]\n",
+ "loss: 0.548388 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 83.2%, Avg loss: 0.540368 \n",
+ "\n",
+ "loss: 0.767534 [ 0/735856]\n",
+ "loss: 0.474996 [64000/735856]\n",
+ "loss: 0.657538 [128000/735856]\n",
+ "loss: 0.388315 [192000/735856]\n",
+ "loss: 0.581206 [256000/735856]\n",
+ "loss: 0.421425 [320000/735856]\n",
+ "loss: 0.494563 [384000/735856]\n",
+ "loss: 0.541493 [448000/735856]\n",
+ "loss: 0.451657 [512000/735856]\n",
+ "loss: 0.382599 [576000/735856]\n",
+ "loss: 0.449485 [640000/735856]\n",
+ "loss: 0.408576 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.0%, Avg loss: 0.455863 \n",
+ "\n",
+ "loss: 0.487152 [ 0/735856]\n",
+ "loss: 0.566136 [64000/735856]\n",
+ "loss: 0.388435 [128000/735856]\n",
+ "loss: 0.435407 [192000/735856]\n",
+ "loss: 0.626423 [256000/735856]\n",
+ "loss: 0.436673 [320000/735856]\n",
+ "loss: 0.599878 [384000/735856]\n",
+ "loss: 0.567672 [448000/735856]\n",
+ "loss: 0.458641 [512000/735856]\n",
+ "loss: 0.479425 [576000/735856]\n",
+ "loss: 0.289777 [640000/735856]\n",
+ "loss: 0.392798 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.1%, Avg loss: 0.449504 \n",
+ "\n",
+ "loss: 0.616642 [ 0/735856]\n",
+ "loss: 0.266790 [64000/735856]\n",
+ "loss: 0.314584 [128000/735856]\n",
+ "loss: 0.314711 [192000/735856]\n",
+ "loss: 0.429452 [256000/735856]\n",
+ "loss: 0.363823 [320000/735856]\n",
+ "loss: 0.594678 [384000/735856]\n",
+ "loss: 0.417127 [448000/735856]\n",
+ "loss: 0.415177 [512000/735856]\n",
+ "loss: 0.406279 [576000/735856]\n",
+ "loss: 0.512797 [640000/735856]\n",
+ "loss: 0.259631 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.3%, Avg loss: 0.415515 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavelet per layer\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'haar'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " \n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ " g.grad = reverse_top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.114821 [ 0/735856]\n",
+ "loss: 1.691283 [64000/735856]\n",
+ "loss: 0.739705 [128000/735856]\n",
+ "loss: 0.878835 [192000/735856]\n",
+ "loss: 0.893373 [256000/735856]\n",
+ "loss: 0.622142 [320000/735856]\n",
+ "loss: 0.729517 [384000/735856]\n",
+ "loss: 0.930510 [448000/735856]\n",
+ "loss: 0.564309 [512000/735856]\n",
+ "loss: 0.820855 [576000/735856]\n",
+ "loss: 0.592394 [640000/735856]\n",
+ "loss: 0.530982 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 82.1%, Avg loss: 0.576240 \n",
+ "\n",
+ "loss: 0.387244 [ 0/735856]\n",
+ "loss: 0.483110 [64000/735856]\n",
+ "loss: 0.544743 [128000/735856]\n",
+ "loss: 0.570393 [192000/735856]\n",
+ "loss: 0.511510 [256000/735856]\n",
+ "loss: 0.335736 [320000/735856]\n",
+ "loss: 0.671059 [384000/735856]\n",
+ "loss: 0.473634 [448000/735856]\n",
+ "loss: 0.559810 [512000/735856]\n",
+ "loss: 0.454633 [576000/735856]\n",
+ "loss: 0.571824 [640000/735856]\n",
+ "loss: 0.626598 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 84.3%, Avg loss: 0.482487 \n",
+ "\n",
+ "loss: 0.422876 [ 0/735856]\n",
+ "loss: 0.769186 [64000/735856]\n",
+ "loss: 0.351542 [128000/735856]\n",
+ "loss: 0.436626 [192000/735856]\n",
+ "loss: 0.628383 [256000/735856]\n",
+ "loss: 0.528591 [320000/735856]\n",
+ "loss: 0.573713 [384000/735856]\n",
+ "loss: 0.517758 [448000/735856]\n",
+ "loss: 0.434379 [512000/735856]\n",
+ "loss: 0.491439 [576000/735856]\n",
+ "loss: 0.494193 [640000/735856]\n",
+ "loss: 0.505279 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.1%, Avg loss: 0.454134 \n",
+ "\n",
+ "loss: 0.439892 [ 0/735856]\n",
+ "loss: 0.459202 [64000/735856]\n",
+ "loss: 0.245611 [128000/735856]\n",
+ "loss: 0.355409 [192000/735856]\n",
+ "loss: 0.490522 [256000/735856]\n",
+ "loss: 0.481495 [320000/735856]\n",
+ "loss: 0.426439 [384000/735856]\n",
+ "loss: 0.641797 [448000/735856]\n",
+ "loss: 0.423894 [512000/735856]\n",
+ "loss: 0.498421 [576000/735856]\n",
+ "loss: 0.344970 [640000/735856]\n",
+ "loss: 0.368346 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 84.2%, Avg loss: 0.466052 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# per layer repeat\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'sym2'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " \n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = 4)\n",
+ " # print(len(coeff))\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ " g.grad = reverse_top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.135039 [ 0/735856]\n",
+ "loss: 1.154176 [64000/735856]\n",
+ "loss: 0.624926 [128000/735856]\n",
+ "loss: 0.605651 [192000/735856]\n",
+ "loss: 0.601686 [256000/735856]\n",
+ "loss: 0.532184 [320000/735856]\n",
+ "loss: 0.627395 [384000/735856]\n",
+ "loss: 0.411491 [448000/735856]\n",
+ "loss: 0.354714 [512000/735856]\n",
+ "loss: 0.393673 [576000/735856]\n",
+ "loss: 0.612208 [640000/735856]\n",
+ "loss: 0.619142 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.2%, Avg loss: 0.445382 \n",
+ "\n",
+ "loss: 0.429652 [ 0/735856]\n",
+ "loss: 0.396769 [64000/735856]\n",
+ "loss: 0.423508 [128000/735856]\n",
+ "loss: 0.576669 [192000/735856]\n",
+ "loss: 0.432909 [256000/735856]\n",
+ "loss: 0.515018 [320000/735856]\n",
+ "loss: 0.375972 [384000/735856]\n",
+ "loss: 0.376615 [448000/735856]\n",
+ "loss: 0.326449 [512000/735856]\n",
+ "loss: 0.360019 [576000/735856]\n",
+ "loss: 0.354862 [640000/735856]\n",
+ "loss: 0.522963 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.9%, Avg loss: 0.383029 \n",
+ "\n",
+ "loss: 0.319733 [ 0/735856]\n",
+ "loss: 0.486813 [64000/735856]\n",
+ "loss: 0.351780 [128000/735856]\n",
+ "loss: 0.327754 [192000/735856]\n",
+ "loss: 0.311207 [256000/735856]\n",
+ "loss: 0.421759 [320000/735856]\n",
+ "loss: 0.486802 [384000/735856]\n",
+ "loss: 0.327473 [448000/735856]\n",
+ "loss: 0.229189 [512000/735856]\n",
+ "loss: 0.395156 [576000/735856]\n",
+ "loss: 0.330383 [640000/735856]\n",
+ "loss: 0.240293 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.1%, Avg loss: 0.374847 \n",
+ "\n",
+ "loss: 0.364167 [ 0/735856]\n",
+ "loss: 0.325380 [64000/735856]\n",
+ "loss: 0.407133 [128000/735856]\n",
+ "loss: 0.229438 [192000/735856]\n",
+ "loss: 0.324557 [256000/735856]\n",
+ "loss: 0.312494 [320000/735856]\n",
+ "loss: 0.250331 [384000/735856]\n",
+ "loss: 0.405609 [448000/735856]\n",
+ "loss: 0.334161 [512000/735856]\n",
+ "loss: 0.305596 [576000/735856]\n",
+ "loss: 0.396855 [640000/735856]\n",
+ "loss: 0.267720 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.3%, Avg loss: 0.357564 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavelet over entire flatten gradient\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'haar'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " flats = []\n",
+ " shapes = []\n",
+ " lens = []\n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " shapes.append(shape)\n",
+ " flat = g.grad.flatten()\n",
+ " flats.append(flat)\n",
+ " lens.append(len(flat))\n",
+ " flat = torch.cat(flats)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ "\n",
+ " \n",
+ " start_index = 0 \n",
+ " for i, key in enumerate(model.parameters()):\n",
+ " end_index = start_index + lens[i]\n",
+ " key.grad = reverse_top10[start_index:end_index].reshape(shapes[i])\n",
+ " start_index = end_index\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pywt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.126535 [ 0/735856]\n",
+ "loss: 1.238513 [64000/735856]\n",
+ "loss: 0.947478 [128000/735856]\n",
+ "loss: 0.758107 [192000/735856]\n",
+ "loss: 0.538468 [256000/735856]\n",
+ "loss: 0.726651 [320000/735856]\n",
+ "loss: 0.523160 [384000/735856]\n",
+ "loss: 0.323133 [448000/735856]\n",
+ "loss: 0.439029 [512000/735856]\n",
+ "loss: 0.406259 [576000/735856]\n",
+ "loss: 0.490085 [640000/735856]\n",
+ "loss: 0.520512 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.0%, Avg loss: 0.449423 \n",
+ "\n",
+ "loss: 0.481995 [ 0/735856]\n",
+ "loss: 0.485922 [64000/735856]\n",
+ "loss: 0.363491 [128000/735856]\n",
+ "loss: 0.604679 [192000/735856]\n",
+ "loss: 0.318160 [256000/735856]\n",
+ "loss: 0.321950 [320000/735856]\n",
+ "loss: 0.355750 [384000/735856]\n",
+ "loss: 0.399116 [448000/735856]\n",
+ "loss: 0.283532 [512000/735856]\n",
+ "loss: 0.527641 [576000/735856]\n",
+ "loss: 0.413641 [640000/735856]\n",
+ "loss: 0.309524 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.0%, Avg loss: 0.406266 \n",
+ "\n",
+ "loss: 0.332639 [ 0/735856]\n",
+ "loss: 0.438504 [64000/735856]\n",
+ "loss: 0.375174 [128000/735856]\n",
+ "loss: 0.325330 [192000/735856]\n",
+ "loss: 0.311181 [256000/735856]\n",
+ "loss: 0.439757 [320000/735856]\n",
+ "loss: 0.357552 [384000/735856]\n",
+ "loss: 0.318609 [448000/735856]\n",
+ "loss: 0.265860 [512000/735856]\n",
+ "loss: 0.534769 [576000/735856]\n",
+ "loss: 0.287946 [640000/735856]\n",
+ "loss: 0.381077 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.0%, Avg loss: 0.377133 \n",
+ "\n",
+ "loss: 0.293545 [ 0/735856]\n",
+ "loss: 0.346547 [64000/735856]\n",
+ "loss: 0.489387 [128000/735856]\n",
+ "loss: 0.438751 [192000/735856]\n",
+ "loss: 0.376747 [256000/735856]\n",
+ "loss: 0.427431 [320000/735856]\n",
+ "loss: 0.381158 [384000/735856]\n",
+ "loss: 0.482535 [448000/735856]\n",
+ "loss: 0.229551 [512000/735856]\n",
+ "loss: 0.455859 [576000/735856]\n",
+ "loss: 0.332654 [640000/735856]\n",
+ "loss: 0.496725 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.5%, Avg loss: 0.358163 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# rerun with alpha 0.2\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'sym2'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " flats = []\n",
+ " shapes = []\n",
+ " lens = []\n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " shapes.append(shape)\n",
+ " flat = g.grad.flatten()\n",
+ " flats.append(flat)\n",
+ " lens.append(len(flat))\n",
+ " flat = torch.cat(flats)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ "\n",
+ " \n",
+ " start_index = 0 \n",
+ " for i, key in enumerate(model.parameters()):\n",
+ " end_index = start_index + lens[i]\n",
+ " key.grad = reverse_top10[start_index:end_index].reshape(shapes[i])\n",
+ " start_index = end_index\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "v,i = torch.topk(torch.tensor([1,2,3,4]), 2 , dim=0, sorted=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([4, 3])"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "v"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([3, 2])"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "i"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.126898 [ 0/735856]\n",
+ "loss: 1.050626 [64000/735856]\n",
+ "loss: 0.652647 [128000/735856]\n",
+ "loss: 0.648297 [192000/735856]\n",
+ "loss: 0.636182 [256000/735856]\n",
+ "loss: 0.570731 [320000/735856]\n",
+ "loss: 0.509262 [384000/735856]\n",
+ "loss: 0.309913 [448000/735856]\n",
+ "loss: 0.538662 [512000/735856]\n",
+ "loss: 0.530801 [576000/735856]\n",
+ "loss: 0.507737 [640000/735856]\n",
+ "loss: 0.422813 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.5%, Avg loss: 0.435624 \n",
+ "\n",
+ "loss: 0.388328 [ 0/735856]\n",
+ "loss: 0.299637 [64000/735856]\n",
+ "loss: 0.420440 [128000/735856]\n",
+ "loss: 0.230143 [192000/735856]\n",
+ "loss: 0.374027 [256000/735856]\n",
+ "loss: 0.279048 [320000/735856]\n",
+ "loss: 0.495672 [384000/735856]\n",
+ "loss: 0.277394 [448000/735856]\n",
+ "loss: 0.395940 [512000/735856]\n",
+ "loss: 0.476103 [576000/735856]\n",
+ "loss: 0.550471 [640000/735856]\n",
+ "loss: 0.431940 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.7%, Avg loss: 0.391393 \n",
+ "\n",
+ "loss: 0.436391 [ 0/735856]\n",
+ "loss: 0.351771 [64000/735856]\n",
+ "loss: 0.352133 [128000/735856]\n",
+ "loss: 0.254270 [192000/735856]\n",
+ "loss: 0.357840 [256000/735856]\n",
+ "loss: 0.368416 [320000/735856]\n",
+ "loss: 0.401375 [384000/735856]\n",
+ "loss: 0.442322 [448000/735856]\n",
+ "loss: 0.538914 [512000/735856]\n",
+ "loss: 0.444955 [576000/735856]\n",
+ "loss: 0.322195 [640000/735856]\n",
+ "loss: 0.493332 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.9%, Avg loss: 0.374818 \n",
+ "\n",
+ "loss: 0.457855 [ 0/735856]\n",
+ "loss: 0.423867 [64000/735856]\n",
+ "loss: 0.274726 [128000/735856]\n",
+ "loss: 0.356364 [192000/735856]\n",
+ "loss: 0.341427 [256000/735856]\n",
+ "loss: 0.301665 [320000/735856]\n",
+ "loss: 0.409492 [384000/735856]\n",
+ "loss: 0.401218 [448000/735856]\n",
+ "loss: 0.616257 [512000/735856]\n",
+ "loss: 0.287706 [576000/735856]\n",
+ "loss: 0.321826 [640000/735856]\n",
+ "loss: 0.423405 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.6%, Avg loss: 0.357925 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# rerun with alpha 0.2 and level 4\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'coif1'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " flats = []\n",
+ " shapes = []\n",
+ " lens = []\n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " shapes.append(shape)\n",
+ " flat = g.grad.flatten()\n",
+ " flats.append(flat)\n",
+ " lens.append(len(flat))\n",
+ " flat = torch.cat(flats)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = 4)\n",
+ " # print(len(coeff))\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.2*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ "\n",
+ " \n",
+ " start_index = 0 \n",
+ " for i, key in enumerate(model.parameters()):\n",
+ " end_index = start_index + lens[i]\n",
+ " key.grad = reverse_top10[start_index:end_index].reshape(shapes[i])\n",
+ " start_index = end_index\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.118358 [ 0/735856]\n",
+ "loss: 1.330925 [64000/735856]\n",
+ "loss: 0.899926 [128000/735856]\n",
+ "loss: 0.894990 [192000/735856]\n",
+ "loss: 0.475845 [256000/735856]\n",
+ "loss: 0.672299 [320000/735856]\n",
+ "loss: 0.728748 [384000/735856]\n",
+ "loss: 0.374176 [448000/735856]\n",
+ "loss: 0.621309 [512000/735856]\n",
+ "loss: 0.562943 [576000/735856]\n",
+ "loss: 0.567177 [640000/735856]\n",
+ "loss: 0.408742 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 84.7%, Avg loss: 0.461047 \n",
+ "\n",
+ "loss: 0.545014 [ 0/735856]\n",
+ "loss: 0.433877 [64000/735856]\n",
+ "loss: 0.513009 [128000/735856]\n",
+ "loss: 0.462199 [192000/735856]\n",
+ "loss: 0.371584 [256000/735856]\n",
+ "loss: 0.380919 [320000/735856]\n",
+ "loss: 0.448126 [384000/735856]\n",
+ "loss: 0.421078 [448000/735856]\n",
+ "loss: 0.531703 [512000/735856]\n",
+ "loss: 0.314307 [576000/735856]\n",
+ "loss: 0.345081 [640000/735856]\n",
+ "loss: 0.456303 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.6%, Avg loss: 0.392272 \n",
+ "\n",
+ "loss: 0.371980 [ 0/735856]\n",
+ "loss: 0.419902 [64000/735856]\n",
+ "loss: 0.344231 [128000/735856]\n",
+ "loss: 0.383977 [192000/735856]\n",
+ "loss: 0.586718 [256000/735856]\n",
+ "loss: 0.524982 [320000/735856]\n",
+ "loss: 0.333949 [384000/735856]\n",
+ "loss: 0.478536 [448000/735856]\n",
+ "loss: 0.346808 [512000/735856]\n",
+ "loss: 0.322247 [576000/735856]\n",
+ "loss: 0.281340 [640000/735856]\n",
+ "loss: 0.373933 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.8%, Avg loss: 0.379021 \n",
+ "\n",
+ "loss: 0.393135 [ 0/735856]\n",
+ "loss: 0.281718 [64000/735856]\n",
+ "loss: 0.488630 [128000/735856]\n",
+ "loss: 0.335369 [192000/735856]\n",
+ "loss: 0.342869 [256000/735856]\n",
+ "loss: 0.293455 [320000/735856]\n",
+ "loss: 0.391644 [384000/735856]\n",
+ "loss: 0.309957 [448000/735856]\n",
+ "loss: 0.277645 [512000/735856]\n",
+ "loss: 0.277113 [576000/735856]\n",
+ "loss: 0.242315 [640000/735856]\n",
+ "loss: 0.292711 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.3%, Avg loss: 0.366945 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " flats = []\n",
+ " shapes = []\n",
+ " lens = []\n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " shapes.append(shape)\n",
+ " flat = g.grad.flatten()\n",
+ " flats.append(flat)\n",
+ " lens.append(len(flat))\n",
+ " flat = torch.cat(flats)\n",
+ "\n",
+ " topk = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat))\n",
+ " top10[topk.indices] = flat[topk.indices]\n",
+ " \n",
+ " start_index = 0 \n",
+ " for i, key in enumerate(model.parameters()):\n",
+ " end_index = start_index + lens[i]\n",
+ " key.grad = top10[start_index:end_index].reshape(shapes[i])\n",
+ " start_index = end_index\n",
+ " \n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Node Training <a class=\"anchor\" id=\"nodetraining\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "id": "9LpgzEw1s-xo"
+ },
+ "outputs": [],
+ "source": [
+ "# From Femnist.py\n",
+ "def read_file(file_path):\n",
+ " with open(file_path, \"r\") as inf:\n",
+ " client_data = json.load(inf)\n",
+ " print(\"loaded the data\")\n",
+ " return (\n",
+ " client_data[\"users\"],\n",
+ " client_data[\"num_samples\"],\n",
+ " client_data[\"user_data\"],\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QBu1kiw8s-xr"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "jI3ixEN4s-xt",
+ "outputId": "ed969663-9e1e-4810-9507-52cdc426650a"
+ },
+ "source": [
+ "# From Femnist.py\n",
+ "for i in range(1):\n",
+ " cur_file = \"leaf/data/femnist/data/train/all_data_0_niid_0_keep_0_train_9.json\"\n",
+ " # test_file = \"leaf/data/femnist/data/test/all_data_0_niid_0_keep_0_test_9.json\"\n",
+ " # cur_file = test_file\n",
+ " clients, _, train_data = read_file(\n",
+ " os.path.join(train_dir, cur_file)\n",
+ " )\n",
+ " for cur_client in clients:\n",
+ " # self.clients.append(cur_client)\n",
+ " my_train_data[\"x\"].extend(train_data[cur_client][\"x\"])\n",
+ " my_train_data[\"y\"].extend(train_data[cur_client][\"y\"])\n",
+ " del train_data[cur_client]\n"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "wvHsSz8as-xw"
+ },
+ "source": [
+ "train_x = (\n",
+ " np.array(my_train_data[\"x\"], dtype=np.dtype(\"float32\"))\n",
+ " .reshape(-1, 28, 28, 1)\n",
+ " .transpose(0, 3, 1, 2)\n",
+ ")\n",
+ "train_y = np.array(my_train_data[\"y\"], dtype=np.dtype(\"int64\")).reshape(-1)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "K8X471SKs-xz",
+ "outputId": "cdf73c06-1323-4e76-850b-16324008d255"
+ },
+ "source": [
+ "len(train_y)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "EpWNELBrs-x0"
+ },
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"wb\") as f:\n",
+ " pickle.dump({\"test_x\": train_x, \"test_y\": train_y}, f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "mAEASHr2s-x1"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "7665.166666666667"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "735856 / 96\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "id": "Am_XlcSSs-x3"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "id": "evAd9ZvYs-x6"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "id": "9_vIFakbs-x7",
+ "outputId": "3a8b546a-186f-4519-8c0b-e853986a8101"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(735856, 1, 28, 28)"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train[\"train_x\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "id": "GPyZ2C8ws-x9"
+ },
+ "outputs": [],
+ "source": [
+ "NUM_CLASSES = 62\n",
+ "IMAGE_SIZE = (28, 28)\n",
+ "FLAT_SIZE = 28 * 28\n",
+ "PIXEL_RANGE = 256.0\n",
+ "import torch.nn.functional as F\n",
+ "\n",
+ "class CNN(nn.Module):\n",
+ " \"\"\"\n",
+ " Class for a CNN Model for FEMNIST\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " def __init__(self):\n",
+ " \"\"\"\n",
+ " Constructor. Instantiates the CNN Model\n",
+ " with 28*28*1 Input and 62 output classes\n",
+ "\n",
+ " \"\"\"\n",
+ " super().__init__()\n",
+ " # 1.6 million params\n",
+ " self.conv1 = nn.Conv2d(1, 32, 5, padding=2)\n",
+ " self.pool = nn.MaxPool2d(2, 2)\n",
+ " self.conv2 = nn.Conv2d(32, 64, 5, padding=2)\n",
+ " self.fc1 = nn.Linear(7 * 7 * 64, 512)\n",
+ " self.fc2 = nn.Linear(512, NUM_CLASSES)\n",
+ "\n",
+ " def forward(self, x):\n",
+ " \"\"\"\n",
+ " Forward pass of the model\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " x : torch.tensor\n",
+ " The input torch tensor\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " torch.tensor\n",
+ " The output torch tensor\n",
+ "\n",
+ " \"\"\"\n",
+ " x = self.pool(F.relu(self.conv1(x)))\n",
+ " x = self.pool(F.relu(self.conv2(x)))\n",
+ " x = torch.flatten(x, 1)\n",
+ " x = F.relu(self.fc1(x))\n",
+ " x = self.fc2(x)\n",
+ " return x\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "id": "bCgW8ClBs-x_"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(735856,)"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train[\"train_y\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "id": "oBGwcwZks-yA"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from torch.utils.data import Dataset\n",
+ "\n",
+ "class FemnistDataset(Dataset):\n",
+ " def __init__(self, training, transform=None, target_transform=None):\n",
+ " if training:\n",
+ " with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)\n",
+ " self.data = train[\"train_x\"][10000:10000+7665,...]\n",
+ " self.label = train[\"train_y\"][10000:10000+7665,...]\n",
+ " else: \n",
+ " with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)\n",
+ " self.data = test[\"test_x\"]\n",
+ " self.label = test[\"test_y\"]\n",
+ " self.transform = transform\n",
+ " self.target_transform = target_transform\n",
+ "\n",
+ " def __len__(self):\n",
+ " return len(self.label)\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " return self.data[idx], self.label[idx]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "id": "U3boC_N4s-yC"
+ },
+ "outputs": [],
+ "source": [
+ "from torch.utils.data import DataLoader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "id": "sJsrQXkEs-yD"
+ },
+ "outputs": [],
+ "source": [
+ "trainset = FemnistDataset(True)\n",
+ "testset = FemnistDataset(False)\n",
+ "\n",
+ "train_dataloader = DataLoader(trainset, batch_size=16, shuffle=True)\n",
+ "test_dataloader = DataLoader(testset, batch_size=128, shuffle=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "480"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(train_dataloader)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "id": "e65Izyv0s-yE"
+ },
+ "outputs": [],
+ "source": [
+ "lr = 0.001\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1487, -0.1003, 0.0990, -0.0245, -0.1023, 0.0974, -0.1139, -0.1425,\n",
+ " -0.1949, -0.0679, -0.0937, 0.0891, 0.0577, -0.1357, 0.0814, 0.1157,\n",
+ " -0.1997, -0.1665, -0.1546, 0.1150, 0.0895, -0.1049, -0.0980, -0.0980,\n",
+ " 0.0729, 0.1947, 0.0421, -0.0365, -0.1470, -0.1679, 0.0286, -0.0146])"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(model.state_dict().values())[1]"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "tensor([-0.0180, 0.0236, 0.1279, -0.1352, -0.1948, -0.0330, -0.1615, -0.0286,\n",
+ " -0.1762, 0.0040, 0.1570, -0.1069, -0.1074, -0.1417, -0.1171, 0.0359,\n",
+ " 0.1276, -0.1534, -0.1773, -0.1639, 0.1334, 0.0518, 0.0586, 0.1466,\n",
+ " 0.1283, 0.0443, -0.0982, -0.1739, -0.0061, 0.1047, -0.0291, 0.1525])"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "for p in model.parameters():\n",
+ " print(p.data.size())\n",
+ " p.data = torch.zeros(p.data.size())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1487, -0.1003, 0.0990, -0.0245, -0.1023, 0.0974, -0.1139, -0.1425,\n",
+ " -0.1949, -0.0679, -0.0937, 0.0891, 0.0577, -0.1357, 0.0814, 0.1157,\n",
+ " -0.1997, -0.1665, -0.1546, 0.1150, 0.0895, -0.1049, -0.0980, -0.0980,\n",
+ " 0.0729, 0.1947, 0.0421, -0.0365, -0.1470, -0.1679, 0.0286, -0.0146])"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(model.state_dict().values())[1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "eqOXilqMs-yF",
+ "outputId": "06799a3b-983b-4f51-a7bd-a901c041bd05"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.158939 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 34.5%, Avg loss: 2.492351 \n",
+ "\n",
+ "loss: 2.274407 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 49.2%, Avg loss: 2.004063 \n",
+ "\n",
+ "loss: 2.080229 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 57.7%, Avg loss: 1.550052 \n",
+ "\n",
+ "loss: 1.220055 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 62.5%, Avg loss: 1.387109 \n",
+ "\n",
+ "loss: 0.547404 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 62.6%, Avg loss: 1.411219 \n",
+ "\n",
+ "loss: 0.666172 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 68.1%, Avg loss: 1.147880 \n",
+ "\n",
+ "loss: 0.539106 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 66.9%, Avg loss: 1.218418 \n",
+ "\n",
+ "loss: 1.057546 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 68.9%, Avg loss: 1.211012 \n",
+ "\n",
+ "loss: 0.315841 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 66.5%, Avg loss: 1.400047 \n",
+ "\n",
+ "loss: 0.659244 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 66.0%, Avg loss: 1.484381 \n",
+ "\n",
+ "loss: 0.437452 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 69.8%, Avg loss: 1.239514 \n",
+ "\n",
+ "loss: 0.675393 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 69.3%, Avg loss: 1.224045 \n",
+ "\n",
+ "loss: 0.409850 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 67.5%, Avg loss: 1.499410 \n",
+ "\n",
+ "loss: 0.942130 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 69.7%, Avg loss: 1.331600 \n",
+ "\n",
+ "loss: 0.193678 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 69.5%, Avg loss: 1.398448 \n",
+ "\n",
+ "loss: 0.120872 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 68.4%, Avg loss: 1.589930 \n",
+ "\n",
+ "loss: 0.099591 [ 0/ 7665]\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(20):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " #print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " #print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " #print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.001\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=0.001)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.158939 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 26.3%, Avg loss: 2.828093 \n",
+ "\n",
+ "loss: 2.478634 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 47.1%, Avg loss: 2.069258 \n",
+ "\n",
+ "loss: 2.099130 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 56.7%, Avg loss: 1.571264 \n",
+ "\n",
+ "loss: 1.343866 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 61.1%, Avg loss: 1.432287 \n",
+ "\n",
+ "loss: 0.783433 [ 0/ 7665]\n"
+ ]
+ },
+ {
+ "ename": "KeyboardInterrupt",
+ "evalue": "",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
+ "Input \u001b[0;32mIn [18]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 48\u001b[0m X \u001b[38;5;241m=\u001b[39m X\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m 49\u001b[0m y \u001b[38;5;241m=\u001b[39m y\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m---> 50\u001b[0m pred \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 51\u001b[0m test_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m loss_fn(pred, y)\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m 52\u001b[0m correct \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m (pred\u001b[38;5;241m.\u001b[39margmax(\u001b[38;5;241m1\u001b[39m) \u001b[38;5;241m==\u001b[39m y)\u001b[38;5;241m.\u001b[39mtype(torch\u001b[38;5;241m.\u001b[39mfloat)\u001b[38;5;241m.\u001b[39msum()\u001b[38;5;241m.\u001b[39mitem()\n",
+ "File \u001b[0;32m~/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/nn/modules/module.py:1102\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1098\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1099\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1100\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1101\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1103\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1104\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+ "Input \u001b[0;32mIn [11]\u001b[0m, in \u001b[0;36mCNN.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 29\u001b[0m \u001b[38;5;124;03mForward pass of the model\u001b[39;00m\n\u001b[1;32m 30\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 40\u001b[0m \n\u001b[1;32m 41\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 42\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool(F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv1(x)))\n\u001b[0;32m---> 43\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool(F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv2\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m))\n\u001b[1;32m 44\u001b[0m x \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mflatten(x, \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 45\u001b[0m x \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc1(x))\n",
+ "File \u001b[0;32m~/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/nn/modules/module.py:1102\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1098\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1099\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1100\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1101\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1103\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1104\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+ "File \u001b[0;32m~/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/nn/modules/conv.py:446\u001b[0m, in \u001b[0;36mConv2d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conv_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[0;32m~/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/nn/modules/conv.py:442\u001b[0m, in \u001b[0;36mConv2d._conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m 438\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mzeros\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m 439\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv2d(F\u001b[38;5;241m.\u001b[39mpad(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode),\n\u001b[1;32m 440\u001b[0m weight, bias, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstride,\n\u001b[1;32m 441\u001b[0m _pair(\u001b[38;5;241m0\u001b[39m), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdilation, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups)\n\u001b[0;32m--> 442\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv2d\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 443\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdilation\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\u001b[43m)\u001b[49m\n",
+ "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(20):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " #print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # https://stackoverflow.com/questions/42704283/adding-l1-l2-regularization-in-pytorch\n",
+ " #to_cat = []\n",
+ " #for v in model.state_dict.values():\n",
+ " # flat = v.flatten()\n",
+ " # to_cat.append(flat)\n",
+ " #flat = torch.cat(to_cat, dim=0)\n",
+ " #loss = loss_fn(pred,y) + 0.02*torch.norm(flat, 2)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " #print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " optimizer.step()\n",
+ " optimizer.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " #print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "after 8: Accuracy: 67.3%, Avg loss: 1.200284 "
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "name": "learningrate.ipynb",
+ "provenance": []
+ },
+ "interpreter": {
+ "hash": "996934296aa9d79be6c3d800a38d8fdb7dfa8fe7bb07df178f1397cde2cb8742"
+ },
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/random files/learningrate.ipynb b/random files/learningrate.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..8045caf05eac5aa85a81910fb935ed6467377488
--- /dev/null
+++ b/random files/learningrate.ipynb
@@ -0,0 +1,6547 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "ZMZYcW3itMzT",
+ "outputId": "f2970f7e-cf26-4a67-e8d3-29bcd1a11775"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "2VftlLfttdT8",
+ "outputId": "48b47fdc-853b-4711-ae95-8c0e64510615"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "ft7BMl1LyWP6"
+ },
+ "outputs": [],
+ "source": [
+ "from torch import nn\n",
+ "import torch\n",
+ "import os\n",
+ "import json\n",
+ "import pickle\n",
+ "import numpy as np\n",
+ "import pywt\n",
+ "train_dir = \"../../\"\n",
+ "my_train_data = {\"x\": [], \"y\": []}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<torch._C.Generator at 0x7f2f8c078db0>"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.manual_seed(13)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "id": "hi0N5rB5xBWn"
+ },
+ "outputs": [],
+ "source": [
+ "if torch.cuda.is_available():\n",
+ " device = \"cuda\"\n",
+ "else:\n",
+ " device = \"cpu\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 35
+ },
+ "id": "6lO3uYsmxNYz",
+ "outputId": "b170b610-f21e-465d-fcd6-b7e6989e73e5"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'cpu'"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "device"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "torch.set_num_threads(6)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Contents\n",
+ "* [CNN Model Training](#train)\n",
+ "* [Optimizer analysis](#optim)\n",
+ "* [FFT](#fft)\n",
+ "* [Wavelets](#wt)\n",
+ "* [FFT Training](#ffttrain)\n",
+ "* [Node_Training](#nodetraining)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## CNN Model Training <a class=\"anchor\" id=\"train\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "id": "9LpgzEw1s-xo"
+ },
+ "outputs": [],
+ "source": [
+ "# From Femnist.py\n",
+ "def read_file(file_path):\n",
+ " with open(file_path, \"r\") as inf:\n",
+ " client_data = json.load(inf)\n",
+ " print(\"loaded the data\")\n",
+ " return (\n",
+ " client_data[\"users\"],\n",
+ " client_data[\"num_samples\"],\n",
+ " client_data[\"user_data\"],\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QBu1kiw8s-xr"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "jI3ixEN4s-xt",
+ "outputId": "ed969663-9e1e-4810-9507-52cdc426650a"
+ },
+ "source": [
+ "# From Femnist.py\n",
+ "for i in range(1):\n",
+ " cur_file = \"leaf/data/femnist/data/train/all_data_0_niid_0_keep_0_train_9.json\"\n",
+ " # test_file = \"leaf/data/femnist/data/test/all_data_0_niid_0_keep_0_test_9.json\"\n",
+ " # cur_file = test_file\n",
+ " clients, _, train_data = read_file(\n",
+ " os.path.join(train_dir, cur_file)\n",
+ " )\n",
+ " for cur_client in clients:\n",
+ " # self.clients.append(cur_client)\n",
+ " my_train_data[\"x\"].extend(train_data[cur_client][\"x\"])\n",
+ " my_train_data[\"y\"].extend(train_data[cur_client][\"y\"])\n",
+ " del train_data[cur_client]\n"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "wvHsSz8as-xw"
+ },
+ "source": [
+ "train_x = (\n",
+ " np.array(my_train_data[\"x\"], dtype=np.dtype(\"float32\"))\n",
+ " .reshape(-1, 28, 28, 1)\n",
+ " .transpose(0, 3, 1, 2)\n",
+ ")\n",
+ "train_y = np.array(my_train_data[\"y\"], dtype=np.dtype(\"int64\")).reshape(-1)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "K8X471SKs-xz",
+ "outputId": "cdf73c06-1323-4e76-850b-16324008d255"
+ },
+ "source": [
+ "len(train_y)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "EpWNELBrs-x0"
+ },
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"wb\") as f:\n",
+ " pickle.dump({\"test_x\": train_x, \"test_y\": train_y}, f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mAEASHr2s-x1"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "id": "Am_XlcSSs-x3"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "evAd9ZvYs-x6"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "id": "9_vIFakbs-x7",
+ "outputId": "3a8b546a-186f-4519-8c0b-e853986a8101"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(735856, 1, 28, 28)"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train[\"train_x\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "id": "GPyZ2C8ws-x9"
+ },
+ "outputs": [],
+ "source": [
+ "NUM_CLASSES = 62\n",
+ "IMAGE_SIZE = (28, 28)\n",
+ "FLAT_SIZE = 28 * 28\n",
+ "PIXEL_RANGE = 256.0\n",
+ "import torch.nn.functional as F\n",
+ "\n",
+ "class CNN(nn.Module):\n",
+ " \"\"\"\n",
+ " Class for a CNN Model for FEMNIST\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " def __init__(self):\n",
+ " \"\"\"\n",
+ " Constructor. Instantiates the CNN Model\n",
+ " with 28*28*1 Input and 62 output classes\n",
+ "\n",
+ " \"\"\"\n",
+ " super().__init__()\n",
+ " # 1.6 million params\n",
+ " self.conv1 = nn.Conv2d(1, 32, 5, padding=2)\n",
+ " self.pool = nn.MaxPool2d(2, 2)\n",
+ " self.conv2 = nn.Conv2d(32, 64, 5, padding=2)\n",
+ " self.fc1 = nn.Linear(7 * 7 * 64, 512)\n",
+ " self.fc2 = nn.Linear(512, NUM_CLASSES)\n",
+ "\n",
+ " def forward(self, x):\n",
+ " \"\"\"\n",
+ " Forward pass of the model\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " x : torch.tensor\n",
+ " The input torch tensor\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " torch.tensor\n",
+ " The output torch tensor\n",
+ "\n",
+ " \"\"\"\n",
+ " x = self.pool(F.relu(self.conv1(x)))\n",
+ " x = self.pool(F.relu(self.conv2(x)))\n",
+ " x = torch.flatten(x, 1)\n",
+ " x = F.relu(self.fc1(x))\n",
+ " x = self.fc2(x)\n",
+ " return x\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "bCgW8ClBs-x_"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "id": "oBGwcwZks-yA"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from torch.utils.data import Dataset\n",
+ "\n",
+ "class FemnistDataset(Dataset):\n",
+ " def __init__(self, training, transform=None, target_transform=None):\n",
+ " if training:\n",
+ " with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)\n",
+ " self.data = train[\"train_x\"]\n",
+ " self.label = train[\"train_y\"]\n",
+ " else: \n",
+ " with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)\n",
+ " self.data = test[\"test_x\"]\n",
+ " self.label = test[\"test_y\"]\n",
+ " self.transform = transform\n",
+ " self.target_transform = target_transform\n",
+ "\n",
+ " def __len__(self):\n",
+ " return len(self.label)\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " return self.data[idx], self.label[idx]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "id": "U3boC_N4s-yC"
+ },
+ "outputs": [],
+ "source": [
+ "from torch.utils.data import DataLoader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "id": "sJsrQXkEs-yD"
+ },
+ "outputs": [],
+ "source": [
+ "trainset = FemnistDataset(True)\n",
+ "testset = FemnistDataset(False)\n",
+ "\n",
+ "train_dataloader = DataLoader(trainset, batch_size=128, shuffle=True)\n",
+ "test_dataloader = DataLoader(testset, batch_size=128, shuffle=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "id": "e65Izyv0s-yE"
+ },
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1487, -0.1003, 0.0990, -0.0245, -0.1023, 0.0974, -0.1139, -0.1425,\n",
+ " -0.1949, -0.0679, -0.0937, 0.0891, 0.0577, -0.1357, 0.0814, 0.1157,\n",
+ " -0.1997, -0.1665, -0.1546, 0.1150, 0.0895, -0.1049, -0.0980, -0.0980,\n",
+ " 0.0729, 0.1947, 0.0421, -0.0365, -0.1470, -0.1679, 0.0286, -0.0146])"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(model.state_dict().values())[1]"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "tensor([-0.0180, 0.0236, 0.1279, -0.1352, -0.1948, -0.0330, -0.1615, -0.0286,\n",
+ " -0.1762, 0.0040, 0.1570, -0.1069, -0.1074, -0.1417, -0.1171, 0.0359,\n",
+ " 0.1276, -0.1534, -0.1773, -0.1639, 0.1334, 0.0518, 0.0586, 0.1466,\n",
+ " 0.1283, 0.0443, -0.0982, -0.1739, -0.0061, 0.1047, -0.0291, 0.1525])"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "for p in model.parameters():\n",
+ " print(p.data.size())\n",
+ " p.data = torch.zeros(p.data.size())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
+ " 0., 0., 0., 0., 0., 0., 0., 0.])"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(model.state_dict().values())[1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "eqOXilqMs-yF",
+ "outputId": "06799a3b-983b-4f51-a7bd-a901c041bd05"
+ },
+ "outputs": [],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(10):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {
+ "id": "4P-VA0vcs-yH"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"/results:128:\"+str(lr)+\".pkl\", \"wb\") as f:\n",
+ " pickle.dump(stats, f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "641-b_VCvT2b",
+ "outputId": "cced38ab-5c04-45b2-faf4-e73327126159"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 60,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "F_OKqiiHs-yJ",
+ "outputId": "65786b88-05f4-42fa-a851-03397ef4457a"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.1: [9. 3.69780584 5.50521373]\n",
+ "0.01: [ 9. 3.98475619 82.61967193]\n",
+ "0.005: [ 9. 0.51492128 85.40642722]\n",
+ "0.001: [ 9. 0.41047618 88.03829502]\n",
+ "0.0005: [ 9. 0.44351858 88.21025672]\n",
+ "0.0001: [ 9. 0.67233266 87.71754375]\n",
+ "1e-05: [ 9. 1.81167539 81.52570279]\n"
+ ]
+ }
+ ],
+ "source": [
+ "lrs = [0.1, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00001]\n",
+ "for l in lrs:\n",
+ " with open(train_dir+\"/results:128:\"+str(l)+\".pkl\", \"rb\") as f:\n",
+ " res = pickle.load(f)\n",
+ " print(str(l)+\": \" + str(np.amax(res[\"test\"], axis=0)))#+ str(np.max(res[\"test\"]))\n",
+ " # print(str(l)+\": \" + str(res[\"test\"]))\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "rADw-XkfKjOo",
+ "outputId": "06c54a2c-f7c2-4610-f879-3e1c2f98543f"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.1: [[0, 3.6898819485246297, 4.914933837429111], [1, 3.695287103771977, 4.914933837429111], [2, 3.691172517592003, 5.505213732544667], [3, 3.6920483804158226, 4.967376059515824], [4, 3.6939517986755845, 5.505213732544667], [5, 3.6917366434742993, 4.914933837429111], [6, 3.695435837910811, 5.505213732544667], [7, 3.6978058357506574, 5.135679004817367], [8, 3.6948341036363623, 5.505213732544667], [9, 3.6921330658768343, 5.135679004817367]]\n",
+ "0.01: [[0, 0.5728003431500958, 81.8086468687115], [1, 0.5517885946725348, 82.61967193121532], [2, 3.984756194857093, 25.844258796268065], [3, 0.5739870879932797, 81.7476675407037], [4, 0.7832032613188912, 75.77779132873955], [5, 0.7142617320772638, 77.80474419171901], [6, 0.6602287095348103, 79.28654186230868], [7, 0.6738644539380036, 79.2719068235868], [8, 0.6469118589079138, 79.77071772669065], [9, 0.6788249858734946, 79.28898103542899]]\n",
+ "0.005: [[0, 0.4834194714537649, 83.79047502896519], [1, 0.466142692822562, 84.33928898103544], [2, 0.4559767278791776, 84.9515214342338], [3, 0.4488265364432298, 84.86493078846271], [4, 0.4554814101660307, 84.773461796451], [5, 0.5149212768315897, 83.05872309287152], [6, 0.4551808235472338, 84.86127202878224], [7, 0.4531376465992325, 85.06494298432831], [8, 0.4589428385362238, 84.83078236477834], [9, 0.4409179601951992, 85.40642722117202]]\n",
+ "0.001: [[0, 0.4104761779773254, 85.89670101835478], [1, 0.36889259526491536, 87.17604731995854], [2, 0.3517718464717292, 87.6992499542655], [3, 0.35526543692939927, 87.57607171168974], [4, 0.3493265717198808, 87.76266845539362], [5, 0.35079776836259874, 87.47362644063662], [6, 0.34534544340812845, 87.96268065125923], [7, 0.35734797465540874, 87.72608085858894], [8, 0.3524193228360457, 87.63339228001708], [9, 0.35447056082407136, 88.0382950179889]]\n",
+ "0.0005: [[0, 0.4435185831906085, 85.1039697542533], [1, 0.37539843543085405, 86.94310628696871], [2, 0.35873422210283473, 87.3797182755046], [3, 0.34818319706667605, 87.93097140069517], [4, 0.34545205666010914, 87.86633331300689], [5, 0.3371337376732536, 88.10415269223734], [6, 0.33852135716659976, 88.11512897127874], [7, 0.33852605533302293, 88.14074028904201], [8, 0.33997187332225476, 88.21025672297091], [9, 0.3402654077747311, 88.1968412708092]]\n",
+ "0.0001: [[0, 0.6723326555745278, 79.72437343740472], [1, 0.5084800024207409, 83.89901823281907], [2, 0.45863669222676995, 84.88932251966584], [3, 0.42524330169194946, 85.90767729739618], [4, 0.4028480564841242, 86.33575218001097], [5, 0.38621816764383715, 86.9126166229648], [6, 0.3782781209337544, 87.11506799195074], [7, 0.3759017101781045, 87.00530520153667], [8, 0.3668581307538772, 87.32117812061712], [9, 0.3569657983208967, 87.71754375266785]]\n",
+ "1e-05: [[0, 1.8116753936371826, 54.021586682114766], [1, 1.3575628893609724, 63.8148667601683], [2, 1.0996285610935432, 69.48716385145435], [3, 0.9349309672803477, 73.7788889566437], [4, 0.8294046315685635, 76.08878590157937], [5, 0.7614829346374863, 77.91938532837368], [6, 0.7032811826737176, 79.30727483383133], [7, 0.6657257149818349, 80.09024940545156], [8, 0.6363296165202226, 80.82931886090616], [9, 0.6094131586890139, 81.52570278675529]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "lrs = [0.1, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00001]\n",
+ "for l in lrs:\n",
+ " with open(train_dir+\"/results:128:\"+str(l)+\".pkl\", \"rb\") as f:\n",
+ " res = pickle.load(f)\n",
+ " # print(str(l)+\": \" + str(np.amax(res[\"test\"], axis=0)))#+ str(np.max(res[\"test\"]))\n",
+ " print(str(l)+\": \" + str(res[\"test\"]))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "HGpNYzG_s-yJ",
+ "outputId": "783622a5-249f-4dd8-d242-fc6dfa47443c"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using cache found in /home/jeffrey/.cache/torch/hub/pytorch_vision_v0.10.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "import torch\n",
+ "resnet = torch.hub.load('pytorch/vision:v0.10.0', 'resnet18', pretrained=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "uZFgT6wss-yL",
+ "outputId": "10f8fc51-abb7-4c2b-f608-85229f3de29d",
+ "tags": []
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "11699132\n"
+ ]
+ }
+ ],
+ "source": [
+ "total = 0\n",
+ "for i in resnet.state_dict().values():\n",
+ " total += i.flatten().size(dim=0)\n",
+ "print(total)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Optimizer analysis <a class=\"anchor\" id=\"optim\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "id": "mRZYP5UNs-yL"
+ },
+ "outputs": [],
+ "source": [
+ "# internal state test\n",
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
+ "old = model.state_dict()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0877, -0.1623, -0.0757, -0.1486, 0.1212, 0.1070, 0.0221, -0.1306,\n",
+ " 0.0798, -0.1525, -0.0297, -0.1715, 0.1039, 0.0143, 0.0982, 0.0428,\n",
+ " -0.0983, -0.0698, 0.1894, 0.1400, 0.0139, -0.0640, 0.0410, -0.0332,\n",
+ " -0.0993, -0.0840, -0.1224, 0.0723, 0.1994, 0.0017, -0.1309, 0.0044])"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "old[\"conv1.bias\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1695, 0.0365, 0.0043, -0.0058, 0.1130, 0.1614, -0.1921, 0.0229,\n",
+ " 0.1472, 0.0111, -0.1327, -0.0368, 0.0536, -0.0637, 0.1539, 0.1022,\n",
+ " 0.1948, -0.1443, 0.1046, 0.1746, 0.1998, -0.0572, 0.0675, -0.1533,\n",
+ " -0.1863, -0.0397, 0.1823, -0.0121, 0.0045, 0.0704, 0.1362, 0.1068])"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "model.state_dict()[\"conv1.bias\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0877, -0.1623, -0.0757, -0.1486, 0.1212, 0.1070, 0.0221, -0.1306,\n",
+ " 0.0798, -0.1525, -0.0297, -0.1715, 0.1039, 0.0143, 0.0982, 0.0428,\n",
+ " -0.0983, -0.0698, 0.1894, 0.1400, 0.0139, -0.0640, 0.0410, -0.0332,\n",
+ " -0.0993, -0.0840, -0.1224, 0.0723, 0.1994, 0.0017, -0.1309, 0.0044])"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "old[\"conv1.bias\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'state': {},\n",
+ " 'param_groups': [{'lr': 0.0005,\n",
+ " 'betas': (0.9, 0.999),\n",
+ " 'eps': 1e-08,\n",
+ " 'weight_decay': 0,\n",
+ " 'amsgrad': False,\n",
+ " 'params': [0, 1, 2, 3, 4, 5, 6, 7]}]}"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimizer.state_dict()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n",
+ "<class 'torch.nn.parameter.Parameter'>\n"
+ ]
+ }
+ ],
+ "source": [
+ "for p in model.parameters():\n",
+ " print(type(p))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "8"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(optimizer.param_groups[0][\"params\"])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "IndexError",
+ "evalue": "list index out of range",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)",
+ "Input \u001b[0;32mIn [19]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43moptimizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparam_groups\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\n",
+ "\u001b[0;31mIndexError\u001b[0m: list index out of range"
+ ]
+ }
+ ],
+ "source": [
+ "optimizer.param_groups[1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "dict_keys(['params', 'lr', 'betas', 'eps', 'weight_decay', 'amsgrad'])"
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimizer.param_groups[0].keys()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Parameter containing:\n",
+ "tensor([-0.0877, -0.1623, -0.0757, -0.1486, 0.1212, 0.1070, 0.0221, -0.1306,\n",
+ " 0.0798, -0.1525, -0.0297, -0.1715, 0.1039, 0.0143, 0.0982, 0.0428,\n",
+ " -0.0983, -0.0698, 0.1894, 0.1400, 0.0139, -0.0640, 0.0410, -0.0332,\n",
+ " -0.0993, -0.0840, -0.1224, 0.0723, 0.1994, 0.0017, -0.1309, 0.0044],\n",
+ " requires_grad=True)"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimizer.param_groups[0][\"params\"][1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# --> yes the optimizer values do not get updates"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "defaultdict(dict, {})"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimizer.state"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# optimizer.state is a dictionary that gets filled during the first step() call\n",
+ "# as keys it has the params and as values it has the internal state of the optimizer (first momentum, second momentum etc)\n",
+ "# stored the values in vals, they are from running the training loop"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "8"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(vals)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vals_list = list(vals)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "8"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(vals_list)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'step': 69,\n",
+ " 'exp_avg': tensor([-1.4137e-03, -1.0432e-02, -5.7605e-03, -1.5292e-02, -1.1802e-02,\n",
+ " 1.1299e-03, 2.1533e-03, -9.7591e-03, -8.8733e-03, -4.7788e-03,\n",
+ " -1.9228e-03, -5.7594e-03, -5.4949e-05, -3.8590e-05, 1.3072e-04,\n",
+ " -7.8018e-03, -6.1446e-04, -2.9151e-03, -3.3301e-03, -2.0083e-03,\n",
+ " -3.0533e-03, -3.5316e-04, -8.1218e-03, 5.7864e-04, 5.8342e-04,\n",
+ " -1.1397e-02, -8.2111e-04, -6.8639e-03, -7.7449e-04, 1.0854e-04,\n",
+ " -4.7743e-05, -9.0613e-03]),\n",
+ " 'exp_avg_sq': tensor([9.6760e-06, 2.4495e-05, 8.1120e-06, 3.0419e-05, 1.0932e-05, 2.3003e-05,\n",
+ " 1.9296e-05, 1.5492e-05, 2.6551e-06, 1.1472e-05, 2.6787e-05, 9.0655e-05,\n",
+ " 8.7915e-11, 3.1380e-05, 4.9582e-06, 3.9729e-06, 7.5247e-06, 1.8417e-05,\n",
+ " 6.9078e-06, 2.9552e-05, 5.0895e-06, 1.6462e-06, 2.1158e-06, 2.8078e-06,\n",
+ " 2.5839e-06, 2.6732e-05, 2.0372e-05, 9.5084e-07, 2.0701e-06, 1.2862e-06,\n",
+ " 1.5106e-06, 2.5722e-05])}"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "vals_list[1] # entry for "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The most feasible solution would be to create a new optimizer and then \n",
+ "# vals = list(optimizer.state.values())\n",
+ "# create new optimizer\n",
+ "# for i, k in enmumerate(optimizer.param_groups[0][\"params\"]):\n",
+ "# optimizer.state[k] = vals[i]\n",
+ "# https://pytorch.org/docs/stable/_modules/torch/optim/adam.html#Adam\n",
+ "# https://pytorch.org/docs/stable/_modules/torch/optim/optimizer.html#Optimizer"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## FFT <a class=\"anchor\" id=\"fft\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(735856, 1, 28, 28)"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train[\"train_x\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "5748.875"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "735856 / 128"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "weights = {}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.124378 [ 0/735856]\n",
+ "loss: 1.539611 [64000/735856]\n",
+ "loss: 0.719579 [128000/735856]\n",
+ "loss: 0.685157 [192000/735856]\n",
+ "loss: 0.778637 [256000/735856]\n",
+ "loss: 0.493262 [320000/735856]\n",
+ "loss: 0.423785 [384000/735856]\n",
+ "loss: 0.531239 [448000/735856]\n",
+ "loss: 0.803173 [512000/735856]\n",
+ "loss: 0.498672 [576000/735856]\n",
+ "loss: 0.453685 [640000/735856]\n",
+ "loss: 0.355350 [704000/735856]\n",
+ "loss: 0.417364 [768000/735856]\n",
+ "loss: 0.462418 [832000/735856]\n",
+ "loss: 0.361217 [896000/735856]\n",
+ "loss: 0.484760 [960000/735856]\n",
+ "loss: 0.360997 [1024000/735856]\n",
+ "loss: 0.353997 [1088000/735856]\n",
+ "loss: 0.378490 [1152000/735856]\n",
+ "loss: 0.376164 [1216000/735856]\n",
+ "loss: 0.375268 [1280000/735856]\n",
+ "loss: 0.570408 [1344000/735856]\n",
+ "loss: 0.295247 [1408000/735856]\n",
+ "loss: 0.257762 [1472000/735856]\n",
+ "loss: 0.609368 [1536000/735856]\n",
+ "loss: 0.423437 [1600000/735856]\n",
+ "loss: 0.363265 [1664000/735856]\n",
+ "loss: 0.393251 [1728000/735856]\n",
+ "loss: 0.353971 [1792000/735856]\n",
+ "loss: 0.279443 [1856000/735856]\n",
+ "loss: 0.532804 [1920000/735856]\n",
+ "loss: 0.364327 [1984000/735856]\n",
+ "loss: 0.310962 [2048000/735856]\n",
+ "loss: 0.306962 [2112000/735856]\n",
+ "loss: 0.391289 [2176000/735856]\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "batch = 0\n",
+ "for e in range(3):\n",
+ " #training\n",
+ " for X, y in train_dataloader:\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " weight = {}\n",
+ " for k,v in model.state_dict().items():\n",
+ " weight[k] = v.clone()\n",
+ " \n",
+ " weights[str(batch)] = weight\n",
+ " batch += 1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.2%, Avg loss: 0.367197 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "size = len(test_dataloader.dataset)\n",
+ "num_batches = len(test_dataloader)\n",
+ "model.eval()\n",
+ "test_loss, correct = 0, 0\n",
+ "with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ "test_loss /= num_batches\n",
+ "correct /= size\n",
+ "print(\"epoch:\")\n",
+ "print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ "stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "dict_keys(['0', '500', '1000', '1500', '2000', '2500', '3000', '3500', '4000', '4500', '5000', '5500', '6000', '6500', '7000', '7500', '8000', '8500', '9000', '9500', '10000', '10500', '11000', '11500', '12000', '12500', '13000', '13500', '14000', '14500', '15000', '15500', '16000', '16500', '17000'])"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "weights.keys()"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "with open(\"vecs.pkl\", \"wb\") as f:\n",
+ " \n",
+ " json.dump(weights, f)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "* 1d on flattend\n",
+ "* 1d on layers\n",
+ "* nd on layers"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# working on the random initialization\n",
+ "flat = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat.append(v.flatten())\n",
+ "conc = torch.cat(flat)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1690046"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(conc)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1440, -0.0482, 0.2070, -0.2534, -0.2413, 0.0336, -0.2401, 0.2761,\n",
+ " 0.2361, -0.2687])"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "conc[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch.fft as fft"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1690046"
+ ]
+ },
+ "execution_count": 29,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft = fft.fft(conc)\n",
+ "len(flat_fft)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "845024"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft = fft.rfft(conc)\n",
+ "len(flat_fft)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-3912.2754+0.0000j, -685.3215+117.2780j, -718.2836-68.4478j,\n",
+ " ..., 33.0949-6.6868j, 49.2176+6.2663j,\n",
+ " -9.9980+0.0000j])"
+ ]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse = fft.irfft(flat_fft)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1440, -0.0482, 0.2070, -0.2534, -0.2413, 0.0336, -0.2401, 0.2761,\n",
+ " 0.2361, -0.2687])"
+ ]
+ },
+ "execution_count": 33,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "reverse[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(0.0004)"
+ ]
+ },
+ "execution_count": 34,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc - reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "top10 = torch.zeros(flat_fft.size(dim=0), dtype = torch.cfloat)\n",
+ "top10[0:84502] = flat_fft[0:84502]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-3912.2754+0.0000j, -685.3215+117.2780j, -718.2836-68.4478j,\n",
+ " ..., 0.0000+0.0000j, 0.0000+0.0000j,\n",
+ " 0.0000+0.0000j])"
+ ]
+ },
+ "execution_count": 36,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "top10"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse_t10 = fft.irfft(top10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0439, -0.0400, -0.0357, -0.0312, -0.0269, -0.0229, -0.0193, -0.0162,\n",
+ " -0.0134, -0.0109])"
+ ]
+ },
+ "execution_count": 38,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "reverse_t10[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(40.2866)"
+ ]
+ },
+ "execution_count": 39,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_t10 - reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "d10 = torch.zeros(flat_fft.size(dim=0), dtype = torch.cfloat)\n",
+ "d10[-84502:] = flat_fft[-84502:]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse_d10 = fft.irfft(d10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0026, 0.0042, -0.0056, 0.0065, -0.0065, 0.0053, -0.0029, -0.0008,\n",
+ " 0.0059, -0.0120])"
+ ]
+ },
+ "execution_count": 42,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "reverse_d10[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(43.7672)"
+ ]
+ },
+ "execution_count": 43,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_d10 - reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 44,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rand = torch.rand(conc.size(dim=0))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 46,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(754.6744)"
+ ]
+ },
+ "execution_count": 46,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(rand - reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 47,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(29442.5273)"
+ ]
+ },
+ "execution_count": 47,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_d10 - reverse, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(27450.3555)"
+ ]
+ },
+ "execution_count": 48,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_t10 - reverse, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 49,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(30432.2695)"
+ ]
+ },
+ "execution_count": 49,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 50,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-3912.2754+0.0000j, -685.3215+117.2780j, -718.2836-68.4478j,\n",
+ " ..., 33.0949-6.6868j, 49.2176+6.2663j,\n",
+ " -9.9980+0.0000j])"
+ ]
+ },
+ "execution_count": 51,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([3912.2754, 695.2839, 721.5375, ..., 33.7637, 49.6149,\n",
+ " 9.9980])"
+ ]
+ },
+ "execution_count": 52,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat_fft.abs()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import matplotlib.pyplot as plt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABgA0lEQVR4nO29eXgc5Z3g//l2S7ItW7Zl+UaWbIExRiaAZbA5AiYcARZiAiFckwkTzp1kd7Jz7DA5vF4nM0N2JjvJ/oZdMIQlk+VwwByGgYQj3NjGlmNiCeJLWLJ8W5Zl2bItdff7+6MOVVVXd1cf6m7J7+d59KiP6qq33nrr/b7fs0QphUaj0WhObkKFboBGo9FoCo8WBhqNRqPRwkCj0Wg0WhhoNBqNBi0MNBqNRoMWBhqNRqNBCwONZtAgIgtFpL3Q7dAMTbQw0OQFEdkuIsdE5IiI7BWRJ0RkVKHblQwRmS4iSkRKcrS/4SJySES+5PPdv4jIc7k4jkaTCVoYaPLJ9UqpUcBcYB7wg3R+LAaDZsx6hYhS6jiwHPhTz3Zh4Dbgl/lrnUbjZtDcWJqhg1JqJ/AaMEdEKkXkFRHZLyKd5utqa1sReUdE/l5EPgR6gDoR+TMR+UxEukWkRUTuc2y/UETaReS/isg+EdktIjeIyLUisllEDorI9xzbh0TkARHZJiIdIvJrERlnfv2e+f+QqdFcYP7mW+bxO0XktyJS69ifEpFvi8gWYIvP6f8SuElEyh2ffRnjXnwt2bl5MY91muP9EyLyY8f760Rkg6mNfCQiX3B897cistM8ziYRuTzRcTQnB1oYaPKOiEwDrgV+jzEG/y9QC9QAx4B/9fzkG8C9QAXQCuwDrgNGA38G/IuIzHVsPxkYDpwCLAYeBf4EaAC+CPxQRGaY2/4n4AbgUmAq0Ak8ZH53ifl/rFJqlFJqlYgsAr4H3AhMAN4Hnva09wZgPnCm99yVUh8Bu83fO8/vKaVUJMC5BUJEzgUeB+4DqoBHgJUiMkxEZgHfAc5TSlVgCKPt6R5DM8RQSuk//TfgfxiTzRHgEMaE/r+BET7bnQN0Ot6/AyxNse8Xgb8wXy/EEChh830FoID5ju0bgRvM158Blzu+mwL0ASXAdPO3JY7vXwPucrwPYWgsteZ7BXwpRXt/ALxuvh5t/v7cgOfW7vhOAac53j8B/Nh8/X+AH3n2tQlD6J2GIXSuAEoLPTb0X3H8ac1Ak09uUEqNVUrVKqX+XCl1TETKReQREWkVkcMYppmxph3dYodzJyJyjYisNk0+hzC0jPGOTTqUUlHz9THz/17H98cAy3ldC7xgmlIOYQiHKDApwTnUAj93bH8QEAwtxLe9PvwKuExEpgJfA7YppX4f8NyCUgv8ldVOc1/TgKlKqa3Ad4ElwD4RecZsi+YkRgsDTaH5K2AWxsp9NP2mGXFsY5fWFZFhwArgn4FJSqmxwKue7dNhB3CNKaSsv+HK8Gv4lfTdAdzn2X6EMsw/ce31QynVimFe+hMME9EvMzy3HsDpe5jsaeffe9pZrpR62mzDU0qpizGEhgJ+kqzNmqGPFgaaQlOBsVI/ZDpu/1uK7cuAYcB+ICIi1wBXZXH8h4G/t5zAIjLB9AtgHiMG1Hm2/zsRqTe3HyMiN2dw3F9i2O0vAp40P0v33DYAt4tIWESuxjABWTwK3C8i880orJEi8h9EpEJEZonIl0zhcxyj/2MZnINmCKGFgabQ/AwYARwAVgO/SbaxUqob+M/ArzGcvbcDK7M4/s/N378uIt1mG+abx+oB/h740DS1LFBKvYCxin7GNGs1AddkcNwVwDjgLaXU7gzP7S+A6zH8MHdg+Bcw97UOuAfDGd8JbAXuNL8eBjyI0ed7gInA32VwDpohhCilH26j0Wg0JztaM9BoNBqNFgYajUaj0cJAo9FoNGhhoNFoNBqMLMuiZPz48Wr69OmFboZGo9EMKhobGw8opSak+7uiFQbTp09n3bp1hW6GRqPRDCpEpDWT32kzkUaj0Wi0MNBoNBqNFgYajUajQQsDjUaj0aCFgUaj0WjQwkCj0Wg0DGFh0NjayUNvb6WxtbPQTdFoNJqip2jzDLKhsbWTOx5bTW8kRllJiCfvXkBDbWWhm6XRaDRFy5DUDFa3dNAbiRFT0BeJsbqlo9BN0mg0mqJmSAqDBXVVlJWECAuUloRYUFdV6CZpNBpNUTMkzUQNtZU8efcCVrd0sKCuSpuINBqNJgVDUhiAIRC0ENBoNJpgDEkzkUaj0WjSQwsDjUaj0WhhoNFoNJohLgx04plGo9EEY8g6kHXimUaj0QRnyGoGOvFMo9FogjNkhYFOPNNoNJrgDFkzUbLEs8bWTp2QptFoNA5yIgxE5Grg50AYeEwp9aDPNl8HlgAK+EQpdXsujp0Mv8Qz7UvQaDSaeLI2E4lIGHgIuAY4E7hNRM70bDMT+DvgIqVUPfDdbI+bKdqXoNFoNPHkQjM4H9iqlGoBEJFngEXAp45t7gEeUkp1Aiil9uXguElpbO1kxfp2BLhxbrW9+rd8CX2RmPYlaDQajUkuhMEpwA7H+3Zgvmeb0wFE5EMMU9ISpdRvvDsSkXuBewFqamoyblBjaye3LVtFb1QB8GxjO0/fs8A2G+kidhqNRuMmXw7kEmAmsBCoBt4TkbOUUoecGymllgHLAObNm6cyPdjqlg76ov0/t8xB1sSvi9hpNBqNm1yElu4EpjneV5ufOWkHViql+pRSnwObMYTDgLCgropwWOz32hyk0Wg0ycmFMFgLzBSRGSJSBtwKrPRs8yKGVoCIjMcwG7Xk4NgJsU5MgG9dOJ2G2kpdnkKj0WgSkLWZSCkVEZHvAL/F8Ac8rpRqFpGlwDql1Erzu6tE5FMgCvyNUmrAwnicZiIFLHvfkDtPrNrOib4Y4ZCwdNEcbp+fuV9Co9FohhI58RkopV4FXvV8ttjxWgF/af4NOJXlZYiAMt0GMWUIBKUM4RCJKX7w4kYALRA0Go2GIViOorG1k6WvNBPzuJ+VAul3IxBTsPilJm0y0mg0GoagMLCSyryUloS494t1hFwCQemkM41Go2EICgMrqcwx5yPA1xqqeeDa2fz4hrMIi/FZSUh0lFGO0M55jWZwM+SEgZVUdsWZk+zPFDBn6hgaWztp2tVFyFIPnHYjTcZY9Z5++vom7nhstRYIGs0gZMgJA4vPDxx1vX9n0z7ueGw1T69poy+qUEA0qmsT5QJd70mjGfwMuRLWja2d3LJsFZGo24O89/BxeiMxrE8FnYyWK3S9J41m8DPkhMHz69vjBIEAF9RVsWlvN30RI8/g5nnTXAXsNJmj6z1pNIOfIScMEhU0emLVdu68YDrNuw9zzZwpOr8gx+h6TxrN4GbICYM5U8fEfaaA430xOxN5TUsHzbu6tGag0Wg0JkPOgdzZ05vwu5gy/nqjiifXtHHbozryRaPRaGAICoMFdVWEA0aM9kZiPL++fWAbpNFoNIOAIScMGmorueeLdQm/98qJjB+aoNFoNEOIIScMGls7efR9/+rYInDfJXWUhQUBysLCTXOr89tAjUajKUKGnAN5xfp2ogmW+6Lg8IkIX5s3Le7ZyBqNRnMyM+SEQTJ3QQx4dt0OojFFWUmIG4eYVtDY2qlj/U8S9LUuHEO174ecMLhxbjXPrG0jGl+4FMB+6M2Jvhgr1rdndTGLaVBY9YF6IzHKSkI8efeCgrdpMFFM1zIV+loXjqHc90POZwD9D7VJug2GlpBpaGmxFWfT9YEyp7G1k9seXc0//3bToAg31te6cAzlvh9ywmB1S0fcg20SEYlm/jyDYhsUVn2gsBSu5tJgLWP9/Pp2u27VYAg3LoZrfbIylPt+yJmJKsvL0to+04tZbMXZCl0faDCrz961Qy7DjQfC/OS91gAPvb01o2MMJvNYMVDo+2wgyYkwEJGrgZ8DYeAxpdSDCba7CXgOOE8ptS4Xx/bStKsr8LbZ3PTFOCgKWR/IT1Mqhj4Jwk1zq3lu3Q76oorSHIYbD6SAtK51NscYzAK8kAzVOlxZCwMRCQMPAVcC7cBaEVmplPrUs10F8BfAmmyPmYwD3SfS2j4bJ3KxDYpCrvKKTVNKh4baSp6+94Kc910+BGQ2xxjMAlyTe3KhGZwPbFVKtQCIyDPAIuBTz3Y/An4C/E0OjpmQ8RXD0tp+697uAWpJfin0Kq8YNaV0GAjBng8Bmc0xBrMA1+SeXAiDU4AdjvftwHznBiIyF5imlPp3EUkoDETkXuBegJqazEpM3zS3muUftyVMPPNyIpIgBrVISbT6L4ZVXrFpSumQD9v+QPRNNsfI9LfazxCcwdRXA+5AFpEQ8D+BO1Ntq5RaBiwDmDdvXkYm/YbaSmaMH8nW/UdTbwyMGVGayWGA/F/oZKt/vcrLnHzY9geSbI6R7m8bWzu5bdkq27/y9L0XFP0k50c+7t1Ca+vpkgthsBOY5nhfbX5mUQHMAd4R4wH0k4GVIvKVgXIijxtZBgGFQdvBnoyOUYgLnWz1X2gzzWBaAXkpBq0qn2RzrVasb6fXVLt7oyrrxM10sNpdWV5GZ09vxmMtX/fuYBtXuRAGa4GZIjIDQwjcCtxufamU6gLGW+9F5B3grwdKEAB0H48E3vbq+skZHWN1Swcn+szY9L78XOhUq/9CmWm8N9fi6+qzulnzzWDQqjKZwL2/aWztZMX6dp5rbCcSzWwi9JZ7ad7ZRWNr54BfZ2uMWfecAOGQsHTRnLSfWpivSXowjCsnWQsDpVRERL4D/BYjtPRxpVSziCwF1imlVmZ7jHR48NXP+GxPMKfwiJIQD1w7O6PjVJaX2aGpMYLnN2SzKku1+i/U6tx5c/X2xVj8UhMxpYpCNQ7SJ8WuVfmtZMFYpScquOgnoJe+0mxPpuA/EaZqy41zq3m20UjSA9i4s4s7Hls94NfZGmNW2xUQiSkWv9TErMkVaR07X5N0ocdVuuTEZ6CUehV41fPZ4gTbLszFMRPxiw/8y1f7MS7NyCMn3nyGIPkNuVBPE63+C2mfdN5cIkI0puxs3kKqxun0yUBoVUEEUZA2erXQFevbeW7dDttc82xjO0/f4/6dd/X7WtNu12QqxGfQBmlLQ20lT9+zgKUvN/NJe1feTCDWGOvti+EM+Yip/ioCQSfdXEzSQRdegymoYshlIPelERy0+9CxjFVcr7oc5OFqA6meFtI+ad1cK9a3s3VvNx9vN8pRxFT6GeGZkKsIq1xqVkEFUZA2erXQA90n7IKL4P877+r3mjlTWLv9IH2RGOGQcPO8aXEaRTr99dnuw/brcHjgTSDOCbz7WB+PffC5rX1WlpfZJqSgpqNsJunB5hgOypATBukQU5knnVnqsnWzBSmHPZDqqXPf4ZCwKwtBlynPr2/nhEMah0j+TOpcYBWZs/rUuUJeUFdFSUjoiyrCIUna37m+wYNOrEHGRGdPLyExxmtIjFya0rDYmkGpOSE6S1L4rX5nTa5IKuyCjs/VLR1EzAJgAnytIT/PBXFO4FfWT7bPxak5RWKKH764MW3TUToM1MKr0EEYQ04YWDdNUDJNOrPU5XQuXq5siH6Dxrk6f3bdDp5a0+ZrPhgovDZdgJKwMQHnepA792cVmYP+InPOY0SVYV9OlXeSi4AAZ7uCTqxBxkRleRkhEcBYCd80t5qb5lbbPoP6qWNYsrKJ3qgiLPCjG87i9vk1cavfVKvhhtpKFl9Xz2tNu7lmzpSE23rPLR9PC7Sc35aPxNuekPRf46jPIi+XYzDVtc3U2V9obWPICYN0BAHAwZ4+1/un1rTZN0M6qma+pHqyQdNQW8nz69ttE4Lf5JjNcVNNWHF9L8KmPd0sfaU5Z4Pce/6XzJzg+t7ZhEfe3UbUbFQ0pnjk3W0s+9N5vvvNNCAgUbuevHtBYMGfbJJubO1k6SvNRGOKkGCfr/M333tho60lRBX8MAOnqvNYvZEYa7cfTLiPTBc1md4jVm6DdY7L1+0gJOKKiLp89iRe/3Sv/Rvx/D7XE+2Nc6tdgsnSyoC0j9XY2snP3txc8DDUIScMQkA6OcUzxo+0Xz+1po3vvbARgPe3HAAIFLaWzFTh3S7bQZlKRU1UgTMbYRWk3c0+DvRotN9xmatB7j3/8RXDKAuLb5G5vYePu37rfe88v9eadmOsvQ3tMl3zlt91+fZlp2V9Qzs1rqiCNz7dy3tb9rvCd731uGIxlVE/p2P+SNfmns3YX93S4fKRRKIKwQhSsNp536Wn8s7m/b5m21yadbznUT91jGuxc9Pc6rT9VM6Q2ZAUrjT2kBMG9h0dkMtmTbRfv9a02/Xda02744SB36SaylRhEWRQppq0U6moN82t5tdr24jEoCRkvM9WCKVqd2NrJ8+u2+H6jWA4Fp2Oy1wMcj8TxU1zq3377JbzavikfaPrvRe/m7EsRTv94vd3HjpGOCSoqMrYoep37a3ztdqnMJ7SZ4XvloQkbvEjkpnjfiB9WtlMyAvqqlw+kpKwEBIhGu1vZzKzbTrnlej+sz7fdegYx02/2Im++MWOgrT60CnsQ8BFp43nu1ecrn0GuSBdM9F/W9mvUl8zZ4qtEQBcM2eKa1unuhoC5k2vZOakCvZ7VmaJmhDE1uiXvOXNuEyloodCISQWIxQynl2U7cooVbudDkV3RyhmTa7Iaax1ovP32+/t82to6zjKb5r3cHX9ZFuwO294r6/jrFPGsPj6+qRmG6cWuOR6d/y+mOedDD9TZCKBbZ3vw+9u4w3TDKLADt/ti6q48RZTsOTl5oRmnkQTXqqxlY12mY2gaag1qsp6fQbW+yC/DzIGE10D5+fO4ymgamRZ3OJkztQx9vWF5M+a8PZLoQQBDEFhkC590X6V2roxE/kMnKn4MeDj7Z18vL2TkrBQEhaiHlOF9+ZxDsrK8jI7Ptq6+H7JW9ZNb61YnZOEE+fKJRI1IyuiMfv4JWEz0iiDVavTsVg1soyfvbmZ+imjqRhR6nKWOmPArciOTM0lXodhOo5Q5z6eWLWd3kiMJ1Zt50oz2/yWRz6yNaeli84iFBJi5nVt3n3YftKZ3zG8WuDytW1xyVDRJGYarymyreMoD1w7247Ecpo+nILunGljeeuzvcRUf/atNTb8SKZ5+pk0nWP125ed5tuX2dQkyjZ4wrrmzpIU1rVYsb7dTqpLpP16x4zzfAH73vFbNDnvSy8dR3td57VpTzc/fHEjMQWrth0gFAolzfbOVVBJLjjphQEEf9pZolVINKq4bX4Np4wdYV/Qp9a0+WbiWhfbbwXiTd6Kqf6bPZVpydqfM5rKFedvrVaDPCDag+VYPO4IG31/ywEEGFba7yz92Zub+WDLAbvNIUkezuncv/NmeGpNGz98caMdHZJpVJRfhNCGHYcw53IiMXjh9+0ox10eiSqeXNPG8rU7fOPV93m0wLKSkEsQ+tl8nefnNUUue7+FmqqRPLtuh91vfgLbu4KcNnYEW5LU3/ILNwV/kyakdnp6axL97XOfUDdhFBMqhvlmQPuRrp/BibeURsi8P7xJdUG0X+f9UhISMJ3RJSGhJBxymZ/AnfBmmeos6qeMdgkq57iNxICY0dfONvktEgspBCy0MAD7hti0pzupA/nGudUsX7eDiCdO0VIPnZFFzkHhzcRNZLaxVgkr1rdzoPsE72zeTySSeJKxcO7POddbcf6WGSfVqtWJnynFi3MV++3LTuO7V5zO2u0H6e2LETKTf6zJPZG25RVk06tG0nLgqGsVlqnTzy9CaJ/HibzjYI/vii8SU/zgRWMs3D6/xu4P74Lg9EkVPHDNbHu16q3J5DU93HnBdJcpUilDE00Vt+80F/2+rTOhIDjfNF16HZvWBO8VZvu6TwQyI3rPe+v+o3Zl4OXrdrB8AKuXWn3oXIzElEKk/76wfFO9fcZCKpnPxHW+UYW15IpEFbfOn+Za1IHR93deMJ1H3mtxCQIBKhxVj1e3dMSFMIdDAkrZwvmef1vH7/64D5WgXEshcw20MACeXNPGivXtzJpU4frc60BuqK1kuWm7tCI4/FZGK9a3uwaFeFbIyeynm/Z088yaNmJAWOC2+TXUTx3jmmS8A8abcBbD0FasOP9Ne7rtOHUReL15D5XlZb42dK+N1JrA/CZMwS2gnMJMgFmTK1JGaDlvzJjCt/R4UKefdzLu7Ol1xRM07erigroqPmnvj3xaUFfFb5r3mJOIsa1Ts1r8UhOAPbGWhITSsBCJKjtqZdOebla3dHDNnCkuE4tfyODhExHC0h8TL0Kckz1R3P6mPd2238APAU6bVMHff/UsHnp7q33c430xHn53G4/+6TwmekqwTKwYFsiMeOPcap7+uM1fcA5w9VJLw/OiFIRCwuLr6m3/0LL3W+yaReAfDVhZXoaIIEoRCkHU3HUMmDN1jO+CZZlHEIAx0VeWl7nGX0kInOumey6eQcWIUirLy+xcEAvvIrHQuQZaGJj0RWJMHD0c6J8ovA5kcNv3E0lv7yrq8jMmxtkv/ZJ7Gls7+cELG227e1QZK7cbzcxRaxu/AePUKN7etM9sSH+cfzSmECCi4JP2LjvKZtbkirj9eVeKzY7SA05qq8q595JT4/rAacv1Ctjla9tcE7Y1EflpHgDVY4fz55fNBPwdcYmigZ68e4FLMwB45uM2rpg9yRYQAsycVMH5M6psf8gHWw9w4Eh/WGlMKZcJIhJVnFU9hjmnjLEFgVPYPf7h53zrohl2v3pDBgV3AlxMwcefd7ji1hNNAF4TkxcFPNfYzk1zq+1ELGvyfuPTvTy1po36qWPsz0vD0h+CmcKM+EbznqTBGakcudk6n0Omj8RLNKbo7OnlqTVtLHu/xW6jt4idc8JesrLJ3pdSJA0pfmpNG//rrc2+EVsxpYx9KUXU9EF96Yz+fAcBDp+IUDGilOZdXa7wWIste7v5xi/WcM2cKXT29BY010ALA5NwSLj/0lO5bNbEpElnli8gGlN2HZRZkytczs76qWNcv1noCF/12j7XtHTQvKuLG83wSO+g+3z/EZfj7uZ50xJmyj7nqCYJxoByOji9Q3H52jauqp8cNwC9mkv9lNEu04ZF28Eelr7ijlrxChKvgP1092E27uyyo6WadnURjSXODNl56DiLX9qIQnxVa280kPM8dh065tpXTMFbn+2ltKTfLlxZXsaSl5sTCiMRGFEatgVWDEOYfrb7MDfOrY6boLfuO8L3XtjIVWdOcrVrYsUw/vPlpxvRVWvaXL95acMuwBiD9VPHJJwAvNFufkQdQQOjh5e6kiof/6CFNodZzGqb04wYiSp+9ubmuKiW3zTvSXjMMlND8nPKZpqI5UUlkUTdx/r4lzc2xwkryyTqPH5IxBX5FlNuU45TK3JqtU6MiDFDk3Cu9CMxw+xotxlYvnYHygwBDpsapfPYL5rX/v0tB7hk5nhbgy9EroEWBiYxjBXt0RMRdhzsoa0j3lzR2NrJ4pea7MEUiSm+98JG1wrs2cZ2Fp7uzoq1Kpo2tnbakSwWvabD8tl1O/jWRTPijjluZJltOumNKjbv7fbNlHU6Bi2UeWzLKaaUOyHPiqcPm9E0IbN+j9fc030i4jK3TK8qtycV7wqm+5gx+VgmJKeAHVEa5k0zIqa3L8YPX2ryXe15z8E4LdN5aWoc3rIPfg7ch9/d5ru/rzVU23Zhv35zEovBm5/tjVv5Wg92STRB7z18nJJQf2z8nsMnWPJyM0uur49LjLR6IFVJZssU8sy6HRzyZM5bhMMhuo/1ccsjq+LCfbuO9bknr6jir369gavrJ2POQcSAD7ceYO32g65J++r6yTz8nrsisNCvHQIup6wlWEpLQtzc0J+IdcI0WZ0zbWxgLWHF+vakiaSrWjp8x5ECO2qv3xQZv10spvhC9RhuOa/G1Z7la9vitg2HhItOreK9BEJ5r8cn48yAv+V8Y3H58ecH2brvSNxv3zODMsKm6Uv7DAqEFUViYQ185/MOEsXTOz/qjcTiMl0t/8JPXvuMRPNOb1Tx5h/3uT4TYKzHEdZ51K3GWoLG6xi02xaDW84zJr/uY32uG3rDjkO80bzHVl/7oopNe7rjIk9C4tYqJlYMY+ehY64Eq8bWTlcsPMD508fZjnHLCfveFiNLFEgpCKw+8DrtnjWd+FZo5OLr6lm+to1hJSFmTqqwTS1e+zhgZ4la5+gnMJwoEgdgCf026cc/aHH5O245r4Z3Nu1zlUiwol6cuxs/qozOnj67LyIxxdKXm31zHZ5a08ZjH3zun9Nhck71GB59v8W3FlPH0fis6u0dPXGTvJ+Qf+Da2ew5fNxeyYLRN60dhnboyrx15D70RmJs2dvt0kbe+HQvb366N3CIqjfD2suk0cMJhQ77jifLJOlX/tp5Hp+0d7Fpr1vLneTRak+pHMHC0yfQtLPLZy8Gp00YSWNPr9tnCC5f0DMfxwsZZ1uUUgNe4NGPUN6POIh4ccNO1/ugWZ0X1FURdvTsO5v389SaNtaapZ0TcazX/YQ2BbYTFAwbb92EUa5trO+a2g/57rMkLMwxzVbWCt8iElVx6r9l9nCuprwC7OPtnf32T2UIkDseWx3n3HxvywGeWtNGY2snD729FYA7L5jOtHHlVFeWx7X1tAkjmTDK3cdTK0dQEjZaHRbD32JNNr2RGI+8u40lK5v4pL2Lj7d38tSaNt5o3kNja2ecgLzqzElxJopUtm4wtI2wz4YVw/rXUiOHlRAytykJC7MmV3DI54aunzLaJQw6jvQiHgPeJ+1d3PboahpbjfHS2NrJzQ9/xPde2JhUEIAh4L2CQDDan05UsV9EztHeaNx2CmO1v6/7BGUlIcJmhI+Tgz5CSNGvXVnjwzpfJ42tnbz1WXKneXlZ2FcQhD2a7kUzx8fvwIGldVptue/SU/vHXgj2d5/g6Y/b4oSBdbolYeGGc6vj+v+86ZX2uHt+fXvKxNiY6tew84nWDIBTxg5n56H4ujXTxrknrKDS+t3N+13L2Ug0fkXoJRwyqk962+EUIDGlqHPUUgLjN0+taWP3Yf/V08JZE+0MWTOk2p4USktCceq/5TR3+g1iyj+rWmGs7i0Hqx+Pf9BCy/6jhgmHxHWjwiFhziljXCtPgJ2dRpkHwGi8h72Hj7tMHwpDq/vFh5+77LMh4OxpYzNSvc86xXAYe239y943+s27slYxY5L72Ef4txxwmx/7zWBurEgTgK8/8pEd8ZKKXs9MdP70Si6dNTFOK0xFJKbispg/P+AfzqowxvyS6/sz5hevbCJiRrTVTRjlGyUG8PvWTrt8Skjgx2bFVQu/cE3vsX/n0agt5kwd7QoPTeVziSlD64zG+n1TS78yxzZxvvHpXt/74IzJFRzri3LOtLH8w79/Gvf9waO9dju2BKiSbI3hmqqRaT/SMxu0MADGjxrmKwy+eq47xC+otPY+djMkktAJaxGNwfG++JWXd5s3Paukdzbt41iS3+3s7LEdzs6bKiyw5HojJK+mamSc03zTnm5mTapg0ujh1I0fmXAisWK817R0xE1E4A4VTTaf1VWVxwkCC6fd1TvBJlrZeyM3QuH+ZzwAdmTJW39MvOq0mHPKmLigADAmj6d8VP7SkpCvaUORuFieH1aWbVBB4CUkRrSUlSviNbmlwltna1x5acJt+yIxOnt6+fZlp9HY2kkI49qEMBYklq/Ii/NeiSn4gedZBAvqquLCNb2oBCrPjPEjWfSvH9hj2KvpexEMbVmZ57NifbttKk3Wd9Y5bO/o8f1+24GjNLZ2smlPt+8CIRHL17ZpYZBvnHHnTryVOFeZK7V0OX3SKLpPRFJul8gpZREOSZzAaDlwlCvOmJhQ0Hy6u9t3wowpw9/Q2NpJZ0+vK3rEHUXRxTnV8ROh1R4rxvuRd7fRetD/ZgiCd8UclE0JVlrhkKBiyhZA/ZnFbSBG6RARUqrsYJgHvu8TVQLxAnxkWZh/u2s+P3ntM9/tZ4wfmXC8eWne1ZXQFxSEcEh4ysymvvDUqrQEgYXzN17/lXe77mN9PPT2VqOsg8MP1dnTyxWeEtOJiCniSnFc94WpCRcKAJfPnuT7ff9nwfo7JBhjI2YEUwi4Et0yRZnntDrN+cOIxMsfOREGInI18HMgDDymlHrQ8/1fAncDEWA/8C2lVGsujj2QeG8eb5hkUD7d3Z1w0krF5Iph7DEnBEFRMbwU6F9djisvdWVB+pHIxPPrdTvsEFdnyJ83XHJDgskrFlO2QMlGEEDqh88k/F2C2XxuzViad3bR47mZnZFJQW3oySYxEfea0Xq13ScaDUjqfPSyv/tEIJ9GIiztKBJTKRcafgi46mwls90Ddoaus82WkFg4a2IgYSAQF96ZTBAAlA/LzZo26ogW6IsqmtO4VqnoPtYXKDTYoiQE9196as6OH4SsHcgiEgYeAq4BzgRuE5EzPZv9HpinlPoC8BzwP7I9bj6Y4zENZHNxojEYUZJ+d+9xrAwjMSOJxcnY8jIW1FVlNGlEosr2CZwwH7QO/sl2figMVdYq55EOJSEj8sXPMZsOiYRBY2tnnCAYEDwSpac3yh2PrebICX/TXVcajsFDPb2uuPV84zwzb1Z9su29m735x32B/W2nThjp8uukSrQD2NAW3PSSDp8mSLbMhGXvt/iGqydiTpJ8k4EiF9FE5wNblVItSqle4BlgkXMDpdTbSilrVK8GBv45eTnAzuQ1+dWq7Vnt73imxt8kjPcJn0wHs8q1nb3a2NrJrMkVSX/jJBqDtwKs+LxEYobGkalGYB8/we/TLWWeKSd8GnCiL0ZpAilXlsaC4EQkRmsaE8hAYC0QspHZx3ojgRynAFfMnuR6XzUydQTfjs5jKbfJhFE50jjAGI+PpOHA39DexYOv+psaB4pcCINTAOeTTdrNzxJxF/BaDo474HhXHK81Jc7CDEIGBUPjOObRDOZMHcMj727LyB5s/d660a3s1XRX+gcLEBNdzCgs85GbkMDo4clNek4uqKvy3U8+sZ4R7udAD0o0qhJG/Hjxmjz9ciO8hAYoQP5EMq91BqR7j6ZyeOeavOYZiMifAPOAf0rw/b0isk5E1u3fvz+fTfPFa5LpG4CVfbp0ejJP39m0jzfTWJk7p5aSsHCBWcPGynxcUJe+o9Evuetkp/t4vDloypjh7DkcfBXbcuAo40cVtm+tCfHF36deIEwePYwvzhwfpxUd7Onj8PHUARQQH7EXRDOQAdICk0XpDUVyIQx2AtMc76vNz1yIyBXA94GvKKV8QySUUsuUUvOUUvMmTJjgt0le8dZDycXKPlu8Tfho24G0nvns/L1SimVmtqoR7258m6hqZiJKwjp30YvfumHnoeMJfQl+NO/qSmhuyhfWo0LbAvgurMmz1JN1lqz2lBdvxN667QdT/qY7oKBJlxJv9lye2Zsgd2igyMVdvBaYKSIzRKQMuBVY6dxARM4FHsEQBMH0xSKgCOb+lBxNY3LxEo25besxBY+8u42G2sq0BkakCDSmoBRabCWKifdDSBy7ni8s/9HkAGGOXccivL/lQJzjPh3/jde30OWjYXnJ1u+UCCmwMFCQV79B1veGUioCfAf4LfAZ8GulVLOILBWRr5ib/RMwCnhWRDaIyMoEuysqvPdtgcdGQspLczfFWfH+ksYud3UFT6QqNIUWW+mYoaeMHZGy/MRAY/mPgmgGiUhHcezpi7kmwHHlhTOT+T1DId/k02+QE3e5UupV4FXPZ4sdr6/IxXHyTazAN2IQFEalSnI0cMeVl/Lgq5+llfVaDOazoUjQWlgDyQdmXHw4i5VQun7YFzfstAtE9vQOjAlosFAzLr6G10BRaK25qPGO4ULbEBNxIoeOrpmTKpLWrvdjoKI5TnY2Z5iomEt2dRkO73yOfecE6Fcc72TihnPzF4Wvb+M08IspLwb8agJlSv3UMZwzbWxavxlELoNBRaH9BdDv49h3JH/OTOcEOJj8UQPBCwGiuHKFFgYaF027uhiZw2QbzeDGMgGmERCUNc6aYLlc6AxG/hCwjlUu0He9xsXWvd20FcGKVFMc2I+tyOMx1/s81+BkJdeJb8nQwkDj4vMDR+MS2zSafJJpBVtNdmgzkcbF4eMRxgfI+tRoBopiyPQ/GdHCQOOiNxKjUgsDTQEpzpi9oY8WBhoXCvjjnsKHNGpOYrQ0KAhaGGjiOLnjNzSFpkwnrhQE3esajaao6M1nHKvGRgsDjUZTVGj/cWHQwkCj0RQVRVr1ZcijhYFGoykqBkF9yCGJFgYajUaj0cIgFQ+9vZVGnR6v0WiGOLocRQr+6bebCIeEHy2aU+imaDQazYChhUEAojHF917YWOhmaDQnDY2tnaz2PA9ZM7BoYaDRaIqOm/7PR4VuwkmH9hloNJqkTH/g3wvdBE0eyIkwEJGrRWSTiGwVkQd8vh8mIsvN79eIyPRcHFej0Wg0uSFrYSAiYeAh4BrgTOA2ETnTs9ldQKdS6jTgX4CfZHtcjUaj0eSOXGgG5wNblVItSqle4BlgkWebRcAvzdfPAZeLiM4z1Gg0miIhF8LgFGCH4327+ZnvNkqpCNAFVOXg2BqNRqPJAUXlQBaRe0VknYis279/f6Gbo9FoNCcNuRAGO4FpjvfV5me+24hICTAGiAsiVkotU0rNU0rNmzBhQg6aptFoNJog5EIYrAVmisgMESkDbgVWerZZCXzTfP014HdKKV2OSqPRaIqErJPOlFIREfkO8FsgDDyulGoWkaXAOqXUSuAXwK9EZCtwEENgaDQajaZIyEkGslLqVeBVz2eLHa+PAzfn4lgajUajyT1F5UDWaDQaTWHQwkCj0SRlVFk478e8/5K6vB/zZEcLA41Gk5SmpVfn/ZgPXDubv/nyrLwf92RGC4OAnFM9ptBN0GhOKhbU6bzUfKKFQQrGlpdywzlTefE7Fxe6KRrNSYE1KTXUVha0HScb+nkGKdiw+KpCN0GjOalQumpZQdCagUajKSp0Omph0MJAo9FoNFoYaDQajUYLA41GU2SU6FmpIOhu18RRXoAkI43GYvSI0kI34aRECwONi5IQDC/Vw0JTOOZM1Tk9hUDf9RoXkZi+GTWFRQcTFQYtDDRx7Oo6XugmaE5i6qeMLnQTTkq0MNDEcaw3UugmaExOxvyrCu0zKAhaGGhchIBTxo4odDM0JoU2mVjuo3wKJV2TqDBoYaBxUVoaYuakikI3Q1MkRGPG/9DJqKKcZGhhoHFRIsKNc6sL3QxNkWBpJvksEbG6pcN+rYVQ/tDCIAnDwu6ReDKMy2GlOsegWCiG8WbJAMljY5xmolih7WQnEVkJAxEZJyJviMgW839czVkROUdEVolIs4j8QURuyeaY+SQcdnfPyTAuaypH8Pz69kI3Q2NSaIFQZi6I6saPzNsx32jeY78u9PmfTGSrGTwAvKWUmgm8Zb730gP8qVKqHrga+JmIjM3yuHmhstwd1RAuwpGZ6zbtOXz8pBB6gwEFDC9wbQZrlX6sL5rxPtIdo79xCIORw7Smmi+yHWmLgF+ar38J3ODdQCm1WSm1xXy9C9gHTMjyuHmh3GMyGVGEJpRojmfuyvIynXRWJIwoCTGuYlhB27Dr0DEAuo71ZbyP4WneN1fXT874t5rMyVYYTFJK7TZf7wEmJdtYRM4HyoBtWR43L+wwbwSLWIHakYopozObMMI+V39ubSVNu7qybJHGj5KQpGX26IspThkzfMDaEwRLCEwbV57xPkakUetqwqgyHrh2tv2+0JpRoanIo2aUsqdF5E0RafL5W+TcTimlSGJWF5EpwK+AP1NK+c6rInKviKwTkXX79+9P81RyzzRPvH0xDsuwQE8kQxXe52rdOLda22nTIJEJxO/jC0+tYlIagrs0JAU32VnHr67MXBicW1PJzAnBfA77j/TS2Nppv596kue8dJ/I3DyXLinnN6XUFUqpOT5/LwF7zUnemuz3+e1DREYD/w58Xym1Osmxliml5iml5k2YUHhL0mTPQCwrwgJuo4aXcPR4ZgPGa2IaO6KEhtrKtENLi9GXki8ShVz6ffzelgNUlpcF3vfwshC7PdppvrFW5pkKpbDA/ZeeSveJ4FntztDSQgvDk4lsZ7eVwDfN198EXvJuICJlwAvAvymlnsvyeHnFWyPl6w3TCtSSxAiCytEtc/h4hMbWThpqK5k9OXji2YRRhbVrF5J0e76zpzfwtif6YhxJYxIdCCqGG0EUXWm0G4xQVMGIyNu0p5u9h08E+l1I3KGlnT2Z+yqKjeJbSrrJtn0PAleKyBbgCvM9IjJPRB4zt/k6cAlwp4hsMP/OyfK4eeFwgW/EIIQEhpfkxq4YU8aqrLG1kz/u6Q78uwNHexl2ktp2R5eX+H6eSFtKp9RCSITQAAT4z55cwdnVwYIEyszrevBoesJAKUNQ9kViLF/bFlhoXj57Eg21/RHqM/IY0jrQZLJky6fWndUdrJTqUEpdrpSaaZqTDpqfr1NK3W2+/n9KqVKl1DmOvw05aPuA83uH7RLg1+t25L0NqTIwvz5vGvVTc1PlUTAmq9UtHXEDN1kzYkoxf8a4nLRhsPG3X57NP3z1rLjr5DeH33DOVDs6JwgxoCYLx20i5tZWsvj6+kDbWpPxjAmjMjqWAnojwUMvLps1Men7wUxpBjN7qV+UxwBxci7nAuJV6cMFyI1PFmctwJX1k5lbE5frlxFVFWU01FbG2bWF5BmoVSPLBlUIoPMyZrrwPm3iKP7hq2dx+/waZk2uiBOWMc/8d8nM8fzs1nNpO9gT+Bgjy8LsOZz7cuL1U8cETixcay6I7r/01IyP1xcNLgy8kWzvbPJ1Q7rI120ZEjilMnOH9rg0/EUWx9MQpNmihUESvCr9QKqsZSUhXzv9qePdKzLnNgpYsb6dVQ6HWzYcNCM5Ont6XTfYvOmVlJWECIsxYMo8K5z/cuUs1nzu34Zi8y1PrhhmC/WQwHm1mQnSkWVhbp9fQ2NrJ7c9ujplvse2A0eB9MxE59ZUMm5k+hNIMkIYi5z93cFs+FY584bayoxNFnVpaBXeQ+wNIAwHqmRFSPpX8+GQ8OMbzmLh6ZkHtuwN2OdOBo2ZaKjjrd552gBV8zxt4iievmcBl/oMtG37j7jeVwx326gFMrbX+42z1S0dLKirosQhDX6/4xCLr6vnL6+axY+/ehZ/ccXp3HDOVKZXlXP/JXXcPr+Gw8f8/SuTM8yByJZEWty+IyfoM2fumIKx5WUZPYC9taOHxtZOnl/f7msGUZ7DW5NqOhVhF86ayNEchxaWlYZYUFcV2H59ycz+MVmZhmCqHVfOF2eO5x++ehb3XXpqwj52CudwKL5IYiF9BnXjR3LXRTOYXlXOPRfPYNbkCjbvDe5LcxKSzHwG+azNpIVBEryruJsGqJrnSDMp5+H3WuK+E48d40QkRllJCMHQJm6cW51xyelwWFyJZyVhY6JoqK1kocNWG4kqmnZ1saCuiqWvNPPPv93Eixt2sb2jh8c/2s6Dr37mO9BvOGcqZ1WPzaht2VBWEuL6L0zBLxLYe3MpYOmis9JO7jl0rI9bl61KuMIu9QgjK6s7aGipYKzgd3flLrS0euxwFl9XT0NtJRMDZjY7x1Y69YkunjmeX901n9vn19BQW8ny+y7kyjMncdqEkS6tc0/3CaLmRYnGFJs8gQsdaTquc8mR4xEefq+F7R09PPxeC7cuW8Xa7Z0pfxcOuU1XV505iXkZaqD5tExrYZCAfJo3LqirYkUCG+7t59e43t9yXg1P37OAv/7yLJ6+Z0HaeQHi+F8/ZbQdJy/A1xqq7UiOCZ7JQjC0ht5IzDXx90ZivLhhp++xCvFchPOnV/KtC6fz4oZd9AUwt3b19LL4paaMknv6oomDeks8jr9Pdx8GgoeWhkPCgroqxuTwqV/th46z5OVmnloTLLrHG+aZCiuc1FqkOGmoreT+S0/lq3Ork5qNHv/AvSC6Zs6UlMcdPdw/oitbjnie+NcXsPbL6RPdPqR3NmeeQDs8jeztbNHCIAEKd/ILPu9zRcWIUl/hc071GB64djb3X1LnMsk01Fby7ctOsyfuhtpKzpySeuI9p3oMw0oN2/+w0hC3nFdj+wKGlYZcms+Nc6vjNJAFdVV2qKGTRKvdLXu7AzkAc8nfXjObZnPiTURp2CgLURIWGtsOEclQFy8NCxMrhvmu3ry23gNHenlqTVu8c14M57KXuy+eQUNtZWBt1KuJJKI3EuOHLzXx9Jq2uO9OqRzhOhevqe1gipj/RWdP5eKZ41lyfb0rPBSgsbWTOx5bzU9f38T2A0cS7AGOewri3T6/htMmJvc5pJPIB1AeMHl0rEcQhwOWE/lsT7fLh9QbibEzw+TBSK6LjyVhyAmDXJ6Qd5AtqKsaEI1hQV2VUQbCs/OKEaU0tnbyxKrttB3s4YlV212p+k5SlQs4beIofnh9PYuvq+fC08az+Lp6bp9f43rvvIEbaitZcn296+ZuqK1k8XX1cX6AubWVnD89Xg1+ccOuhKupoP04eniJ774TsWJ9e8oHqi+cNZGLZ47nS7MmEstQEFSPHc4z915gC80gvNa0Oy5aRin4ePtBrjqzv6xXSPqfAxz0ecAT06hhFI35azS3n1/DbefX2NcmFlOuBVBfisiWV/6wmw+3HmDpK81x49TSKmPKiLRK1GN+5SfmpAidjnpDt1JwIuAEe6S3XzAJcMt507jizKTl1xKyuyuzqLB8WigGRr8qILkMxPKq9A21lUwaPYw9AbMpgzB7coU9CS86eyovbthlf1c1soyfvbnZvon6IjFWt3T4rrp+l2IFvm3fEW5btgpEiERjrN1+EIClrzRzoi/Gqm3GTX/7/Bp7n0tfaaY30r9t064unmtstycFAUodJoGbH/7I1ybvJByCW88zjvGkz+rUy4K6KsrLwkBqWy3Ac43tSSM+SsLCO5v2EYkqwmGhtCREJBIjFBLuvngGL/9hFzsPxd+4IelPpCoNCz+/ba59HZ68ewEPv7uNNz7da28/e8poPvbYl+unjGZ9W/x5nOgzTG8lISEaU4RE7IVIUDPN1DHD2dmZevUZFiMrOBKJxd0rleVlLKir4lnzGodNH5JFKhOXJWR6fcappVX2RWKICDGzM0MYfWqNkw3tXXYWPMBTa9pc94QfQcIvrUk1JNg+ilR84ZQxfLz9oNEXplYQ1NcSR4YL/JI8Og2GnDDIFX720sbWzpwKAoBZjlBRr439lT/stm8wod+O7OX59e0p1UkF9EaVXbyiLxLjtabd9kQUiSkWv9TELFM4OVdyvX0xFr/U5FpRhoCLZo7nu1ecTkNtJU+tafONfAhJvNP2xrnVcY7CRLzumGD9qB1XTtvBHrtdfZFYwnDECaPKOLem0t5nJKq46syJjK8YZk8WTkEQDkFDTSUnIjHGjCjl/S0HAGPFvGlPt8tM9+ifzuOpNW281rTbtnN7hUHFiFJO+EUeAW99ttfup0hMsWRl/7U4c0oFn+5O3F8hgdMnVdDY2pky+iQcDrHk+no6e3r5ZMchuy+skNNNe7qJWnkBnsJLyTSgc6rHsKHd0HpiKl6rtrTK15p2Uz9lNI9/+Dl9UUVJWFg4ayJvfLoXBUSjbkHyWtNu76HiSTFfloSMIIHOnl66j/X5Bmr4Mb+uir+44nQefncbv/vjPp7+uI2SkBF0kSh1YvTwEs6ZNpb3zLFit6EkxDnVY+LGRCpOpJGjkS1DzkyUK0I+Ejkbn4EI3H9JHWM9D8x5ccMuvv/CRhpbO+NuIOfkq6yd+JDOoqPU9BGUloS4Zs4Ul104pvrNAtZKLixGX8SU27RQUhKyBQHA4x9+7t82T+OiMXjk3W109vTmRAWeNHqYq10KwyFf4hOg7ReZojCE6dMft/GIZ5KYPHo469sO8Ul7F+9tOWAfJ6pg8UtNcaaQ2+fX2BE0b3s0NWtxcct57oAAC+8k3htVrFjfzlNr2pIKAuu3FcNKKCsJ2Te04D9HRszyELsOHWPhrIkMN31IZaUhKsvL+OGLG4maGlDEYyZSCaryCfCHdrf5y3v+lqb54dYDPP7h50TN8RTDMNtZvqzSErc24udAHlfe72MrDQvDE2TpCvDFmeNZft+F3D6/hm9fdlrKgnmlYSEsMNwMwd20p5u3PttLNKaMhVFUUT02sUn2S2dM5MNt8fPE1xqqMwpNz2do6ZDTDETSf3h3SIzojz5HpEwsquJU3XQdVU4WnT2VihGlLDx9Qpza++SaNp5dt4Ob501D6J/cw6bZwJ6Eov5mopvmVvNUAJPL7MkV/PirZ9m5BNZ+Fr/UREwpyhw3YkNtJU/evYDVLR1UlpexZGUTvQ7tI+bt5ASdHg5JnIN2XWunPQFYmkmm+NXMOXwiYtb0ce9ZKRhfMYyysNAXVbYD2NKAvOzuOp7wZrQEp/NaNLZ22n37uSc/ZNzIMtvv8v9Wb085wYMxmQVaGQOrWjpc16uzp5fK8jJ+8OJG+xwEY/L9pL2LT9q7KCvp1xIWmBFtTgVTpF8TbWztpOOovwPZr3a99/yfX99uX2vnOLLClq22O8clGAK2reMoj7zXYh+js6ePkBh1jO679FQeeXcb7T6mvdKwuBYsja2dPJukpEwIuHneNE4ZO8I+b+PecG/XmiSL/KUNu+L6IiRGaPGsyRX8em0b6SQVj8hjza8hJwwymVnqJoziWxfN4O1N+2y7b4z4yT94WCD8aNFZvPD7dnYc7GFBXRW/ad7Dyk92JSw81htVbN7b7Wr+dV+YQvmwEp5rbCcajcWtmiwaTAduKhV0rjkZQb+WY5VT8LsRrcmrsbUzrlsjUcXSl5tZbDqXv3VxHd97YWPcMa/7whSadnaxdf9R+7ODR3tZ+kozi6+rt1X3RLb6VNRNGMXnHUdttd2KgIr4qNfhsHDT3Gpumlttn++mPd2ERFAq3qmabFVW5rkWVrRMr5kHMqPKHZN/8KiR3b1pT7dLEAgw28cMZEVwbdrTbZunknHgaC/Pr2+nfuoYe3J/o3mP6xxOnTiKbfuOuExqTbu6OMV02npH5uVnTIwbL0EZN7KMh97eavfRs+t29JsYPaZDoX+s+XFl/WQe/2i7ndynMLSz3/1xH/ddeir3XXqqrzlRAZv2dNvXenVLR9Lw0DIzos5qx0Nvbw3sX3Ae00tMGb65J+9ewPL7LuQHL2zks4Bm0sl5fLjRkBMGmawyt+47wpKXm7m5odoeqCGJn/yNzFxSSnYrUeyTHYfoiype/mQXMVP1jltRO+iNxFyawSt/2M3y+y5wTV6Jbpi/vWY2tzzykW/bnI5e76T15N0Lkt6IYKzq/G6iT9q7uOOx1Tx59wLb8fyjV5o55gjw//zAUX7ytbPj2tYXidHZ08u3LzuNxtZOHv3A38wEhtPX8omUhAWlDBNaqWlv/t0fTbu3wJLr65k1uYIVjpWohdX1TiG39JVm02lrXZ+EzTCOAZxVPYZbzqtx9ZnTx9Jn9q0TpYxtvJPqF6rHsPj6em57dLU92Y0bWcbXzZwP6xjL17ZxtDfqmsyd7Ow8ZjvkQ2I4Hvs8JzOyLExpWOyVeTgsLF/bRjTWb1e3nLylJSHuc9QjWlBXRYjEARrnTa9kfdshuy83tHexrrWTshJjgrW0Q8FY0b+7eb99nPqpY2zB4TcOV7d0+Ar3qGnGWlBXRVjin8/RF1X80NSMSsPCty6akXB+ONu8Dl6n97DSEL3meHa24IZzptJxtJf6KaNdWgv4+8ms4I8FdVVpZTFfMTuz6KVM0D4Dk95IjP3dJ2w7uXflB8YkMn28f8xzWUn/uioSVTz+QQu9ZlKSZYNNxQV1Vb42fG9egR/OLE8ngjHhWOGh3kkryIovWdt7+2IuLeOLM92RPJNGD7fbdvv8/ryG0hLDRv3Q21t55N1tCVdgglGZ9fb5Ndwxv4alX5lDWPpt4u9s2mcLmZgyIp4sE5f3qWJRjw3cm0R3+exJSaM3KoaXEA4LG3d2xYVPOn0spSUhLvCMnZKwYXLx2sAtobLk+nr7Zjx4tJeH32uxTX+3z6/hpe9czE9u+kJc7L9fa2PKTIjzdOmM8SN5+t4LuPLMSZxdPYaZE0bZGlUkZtj5vQmNFg21ldx7SZ1rf+GQmWQWFr56bjXO57JFov1jTIHtzwiHhFPHj+TS0yfwheoxfOvC6Sx9pZmfvr6JOx5b7Rs6bfWt91ytgIrVLR2+taFC0n/v9UYVq1o67DwKKyDDyqPxCgLrnJ+8ewF/9eVZPPsfL+QfvnqWXWLjZ7eey6/umk9N1UjX/SHArefXcPv8Gq46c5JrTCRrayKChhbngiGnGZQ5Vj7pMr5iWELbpUXd+JFs3RefNFNeGqY30u+cSvcB4lZs+dJFc+zVTEmC6KFENNRWcs60sa4QR4Xh3Nu0t5lZkytcIX6JzE5ebppbzXPrdtAXVYRDMLem3yTlNactnDXRpbJbZS2sVa6l5VSWl9mhq8mkTTgkcap7xPSjRGKKlgNHXdsfMMtDNNRWUjOu3BX95Y0Q8/bF/ZeeymWzJtpOVC9HjkdcJhanz8DpY7Hs704WzproWulbUUeWRtXZ0xu36n6tabf9vXWMpYvmuPwAQNyq2KkZOAWC5UB/f8t+Qwh6znHf4eNJtUQrOVJhrCJvOa/Gtq+vWN/uEsrhkBBGUWpqBnOmjrEj0pzRPE27DqOUSho6bUUiOa9LSGDpojn2tl4PUfXY4Zw5dYxrLB49EaEkHLJNrpaZMpnG7eyPhtpK1/UAQ2NzojD8A84Qbe98EnSOSjcDPFuGnDDIRAwIhhppTTrJVuD3XXoqbzrCAC16PLUPyjwlncNmoHppSYjzp4/jDzu7XHHMzok5HA4Ri8Ts6CG/AZWIbh8hpOi/0b592WkpBZ6XhtpKnr73ApftdZ0Zxug1p1kVTxOZ2qz+fejtrYGcx3dfPAPANiNUlpfZfR9TRmSJk/GOOPCxHp/PPM+19U7gYEzAidpkfW6Z3ZLdqN5VrDM+/fb5NXGTip8J0qtFWBVl7/1iHY998Lnt9F98XT1Nu7qMEiMen4Fz4h1RGrYL68VUfBsTRTo522gHaAguIf3Iu9tc215+xkTOnjbWHmOrWzp8k92iMUVJyHheX7I+7ezpjXNuO8OyvYEjVaOGsXDWRN7ZvN82v7UcOEpJSLj1/BpudLQ9E6x70q9I5NJXmu2wYO98Yt1LK9a38/Inu+g+nji6Kd1AmGwZcsIgmqZWMLliGN+4cHpaE+OPbziL77+wMa5Gj0VJWFyx7iGBey6eQcWIUteK+OPtB+NWJw+9vZVINGbHXK9Y327fwE4bvx8PvvpZXHikUwg5I4XSvRG8v0mkXQTVPCrLy1IKAsGIDHL6OLzlGcZ4JnyrIBy4BQP4V511+g/ueGy1LaDsFbC5yo7RXxogFJK4jG2rlLV13kuur3f1Q6r6UZYp7eF3t7Hv8HFuOc8tMLy+nqWL5qRc1QI8+sHntgnuzc/2UhIS1+r4zgum07z7sEtLScSvVm13CeJfrdpu952z7EhpWLjv0lPjIvH8rrc3oinRufiFXf/szc1894rTjYcxeXa+caehDS+53sht+HDrAWKmr2nq2BEZCQJLADjv4ZKQxPkInObXZIEZv2/tTOpIVubvsxFa6TDkhEG6KRp7uk8EFgQW1k3zQ1PtdTJqWJgLTx3Pm5/1q6cxBU+s2m6vQu1kLocT1cI7mQoEykD2ZsCCMYkFnTTSwbuiTrbaTnRMK88gmUBQGGYfZ395nW9dSTQRp2Dwe+/E6T+wEuqumTPFXnFv2dvdH62lVJzG4yxl3RuJ0byri6fvSU8DAzjHsZr2a581Drzjxo/n17e7xqc1Gd5yfn/4ZDpjwltw7c3P9vLQ21vZdeiYy0F887xpcfv1Xu+zq8cw55QxgVfofuPlw60HWGsuqCzHuKXtOPvpu1eczlpHJvGuQ8ds34QzFDdZfziFsdBvlotElSv9J+Twh/kFaji1/CAP/ckmnD1dhpwwSDXB+PHwu9t49E/npfUbSyD842uf0n28v4bJkRNRfvfHeDOSNZF7zRzdx/pckRR+posV69sTrrStQXrcp0Tnl86YaKuruSaZduH3ndfUFVQzcEb4xFT8A9Ktkt5+/ZPKZOXEK4S/e8XpQH+5DmdbvWUaIH7MqQT9kIhEUV6J2hfEluxtk2XeuinFBJzILDl9XDkberrs90d7o/z09U1x2oZfcT3v9fZqPslobO1k56FjdukQTJOQNeE37eoyTarGxBwOCbFYvzZs3VMr1rfzXKORYPjsuh0gYju4Q2bQSCLN2ymMXX0aEjsZLwRcdNp4W1vxC9SwrnFJSAKVtggazp4LshIGIjIOWA5MB7YDX1dKxYcDGNuOBj4FXlRKfSeb4yYjlCRVPBH7Mni0YGNrZ1wiloX3+E4b8+qWDnuSE3DZfp1hns4BmWylbQ06P978bC/vbdmf1LSUSxJNIo2tndy2bJWd6PX0vRe4VnqCcS/7BRRNNEtFWCv2GR4H/i3nJc6TSGcC9dNoHnp7a1zJbsFd6tvC6WS3/E/p4Dd5WMew+jWIw9OJVxO68sxJceYbL8mEkjeyxarVFETbaHYU6AthRH0lCyf1a09JSLhtfg31U8ew9JVml/Zs1cuKKZCYivMLWH4LK8rJuG/dWlMizRv6x5JzYSAYC673t+x3LSJsR7Fn7LmsAlHlmyjnZTBpBg8AbymlHhSRB8z3f5tg2x8B72V5vJSMLCvhcBKnjB/eMMAg+CWwWJOaZWOOmtE3t5znHpjDSt0Fu1INxGQrTGuQ+mkGqfabS5w297BpnrJWfivWt9tC0yqzcNPcarsfrMiOpl1dPPNxf40jhREB4tzOivjxRuMk6rd0nOXefrb6trfPKOpmrR79Jnqvkz1IfzuFZyLBlUpjSIZzVSnA2dPGpvxtMqF0zZwprgS40rAQjSpExBVB43eezszfUFh4rrGdSDT1Oa1u6eivnxU17P3eREmA5Wt32KaqmML3WdPWE/ycCzinb8gZ7pzI/GlpF5YWdP+lp3L/pafGXfdEY88rUFIxaDQDYBGw0Hz9S+AdfISBiDQAk4DfAOnZY9LkS2dMTFnl0EsmsbwL6qpcCTxgDSojueXxj7YTRREKheJWKM6CXU+s2p6W6u/FGnR3/XIthzwmFGuA5yM8zXXTeoreeaNWhMQ3y+8+2+sKB92w41Dcdn4hfonIxFnu/K23xEPQMMRU+E3yfv2RbHJOhXNVqQi2ykymTVmlIX7TvIer6ydTUzXSLmXijKDxsrqlw+VTqJ8ymo07uwKdk9O85Axj9vb10kVzXMUULX+CU9A01FZy87xp9sN9nL4hq3yH5Rj2E1LWMW+aW82K9e0pa2v5RRI5BUoijd57/vkiW2EwSSllFU/ZgzHhuxCREPBT4E+AK7I8XkrKh6V/Sp/sOOQqmxsEayW49OVmPnEU6YrFFM27D7sigrwqv7M0dLqqf6K2eGsezZwwksljR3DNnCl5MREtMBPm+ldn/QlzN86ttssiOyNr/CbPG845xRUOeXX95Ljt0gm1zZZshEky/CZ5v8TCTHwFFun4TCySaVPWszV6IzGeWLWdm+ZWB9JsnRpWKCRcUFfFpr3dgc4p6DnMmlzB18+bRvPOrqSC5kZzIvcz61hmwSBCygoYsHwPQbQcq38tgfKNX6ympze5QCgqzUBE3gQm+3z1fecbpZQSET/t58+BV5VS7d7n+foc617gXoCammArPy9bM3hg9eufZm5brz9ljDn5G6duVQNd65M/AJlFhQRh5qQKl/P8844eth04ytrtBwfEieydkBtqK+3VmV/Ru0SRNd79XFk/2Q6HDIeEK+snxx03U7NJtueYSyHkN8n77T9dU1eqYwQhkQD0jl1F4hBj7/4WX1dvj40nVm0PvAgKcg5On1Q4hMuZ7VdFwNufzpDRIOfj6gfT96BIT3Ozgih6ehP7DYQiSzpTSiVczYvIXhGZopTaLSJTAL8nrFwAfFFE/hwYBZSJyBGl1AM+x1oGLAOYN29eRikXflUsg5DOhWxs7bRVvUjUcGxddeYkJlQMs01CVn0cr/jLZqWXDKuOiuWLsB800pd7n0GiCTlI0btU+zFixi2nQXxl0GzMJunw1Jo2l2BbfF19UhNCungnJSChkMtUO3GaJHOhIXrHrrfoX7L9d/b0urSIoIugIMLQ6ZOKxODKMyYkDNG19unU1J39HkRIOfshHBIQSVpIMhH1U8ckLc6Y55yzrM1EK4FvAg+a/1/ybqCUusN6LSJ3AvP8BEGuGDeyDPYfTb2hg3Rs697kJDCiKc6eNjZucFuq5Ir17a5IIctumMrmmA7Om8b5AA+/6qt+55TOyjPZhJzOxOW3n1TCckFdlV1u3C/EMxc0tnay+KUm2+TVGzEeBJRrIeTsq3RMFOmcx5KVTfRFFWtaOrLWEBNNzEH2mc0iKNWY8t5HEyuGBda2M9HU/QR5Jpqbt3RLovblw8wL2QuDB4Ffi8hdQCvwdQARmQfcr5S6O8v9p423BEEqrMcwBk1+8RY3M/YRX0Mo1QrWT1Bki3XTPPT21sC24kzMLrnSbvz243XaWvHZrjY5NIeBwCqdYBESSWr6ywUDoTF6o7icJcczJRstJVNzVyoS+aSCkCtTWibnk8ofMKhqEymlOoDLfT5fB8QJAqXUE8AT2RwzFd4SBKlQirTS073hhoDvE8iSDbKBNHU0tnayYcchRIQQbtu9H5m0JVc3dqqVpp+QsiJTrPj2gVg5WSY3y+FphckmMoHlgnT7NIg25x2VzpLj+VptOsmVM97PX5VJtrfVpoESUqlYUFflW+7a4itnT81re4ZcBvJNc6tZ/nFb4DKxIUm/MuiTdy/gZ29u5gPzUYgRnyeQJRtkyQRFNk5Ky5Fm16v3qaHjJddOxnQJ6qy0+negfC7eNiUSUrm6OZ3PS3bmSgT1WQXR5qwVszOEMV95JwNFsudxZLMoSfbbgYxec+Y9VJaXcKinvzLuq017+EaaUY5ZtSUvR8kjDbWV3PPFusAPvb774hlpd3ZDbaUrASem/O3yiQZZoskm20gZbyJcLBZfQydoW7LBb6JLhfOGA9h56JhvVMhArKD9yOXE7+WpNW32U+GsMRS0nyC4NmetmL2JUvk0PeSafAUQWAxk9Joz/yIsMOeUsXzgSOrLt+AecsIA4JU/BE86S/WA7EQ0OdLrwahrno6Dzm+yyXagexPhgt74uV7xpjvReUsOWHHbicoN53oFnW+8zzX2PrcgFemW2WiorQwc9VPs5EMzdDKQwsd7LtfMmcKalo60799cMeSEQWNrZ6CaHxaZuiCth6hY5MIem+1Ab6jtr5UukHXN9kzIZKJLFLcdiSrfsgJBCVLvpxCTo7esg/e5BanIRJsbSE0nn+RKkw16/QdS+PidizMkPd/375ATBlb0SRDCIdIuKGbh56jONqY/FwO90Dd9JhOdX9x2JGI46P3KCgQl0Y1caI3BEo7pmtIsnGa4oTDBp0u2YzxZHS2/Yw2kg9l5LpaASlVVdqAYcsJgQZ3/w7H9CKXIiE6GX238IDH9qSj0ZJ4tmUx03hsO4GdvbrYfSJKpep7oRg6q+g+k9uD3tLMgZOtvGKqkc62S1dHyYyDvSSuB9UD3Cd7ZvD9wWYuBYMgJg4baSi6fPSllMgcYZohMV/Jvb4pPthbyW0vESyFNH04ymei8N5zzgSTZqOd+N3LQEgdD0d8wFEl0rRLdD8nqaOW73bc88pHrUadQuIivIScMvI/gS0YoJBmv5P2egRC0MuRAUKyTVzbcOLd6QGynQVT/fEetBCVbf8NQxO9aQfLyHonqaOWTFevb4wSBkH/HscWQEwbe8EoBZk+p4NPd7gJ2xgO0k5feTcYt59XwSftG12chCqcZOG+I3r6Y/XzYYpjA0sUr2NLJKA1KKtU/31ErQcnW31BIBkpz9btWqYT5QCcRBsFrpA4J3OYTPZcvhpwwsLL6LHmgIE4QWF/EyHzVZ9V2f3HDTvYf6Y176Hy+8WZGZ+N4LSSNrZ387M3NBV+VD7TjMBsy9Tfkgkwn9IHUXBNdq2IU5k6cSYHhkPCjJI7sfDDkhEFQn4Hz6UaZDBRnbfeQGKWsbzmvpmCThnVD5MLxWgiclWCdz6Ut5I082J35uSabCX2gzW7ea5VKmBeDWdVKCiyWBceQEwYA9116Km/9cZ+r2JgXof/h1ZlcBOfgjin4Q3sXm/ZmZnLKFQ21lTlzvOYTv0qwzoeLF/om0RhkM6EXwuyWTJgXi0+omBYcQ1IYNNRW8iPPY/C8hEOSVZx2ZXmZq8CU9+EWhYrsKWbzRiK8lWAFKCsNaUFQZGRbhrqYxmXQcymWCL18IGqAygBny7x589S6deuy2odlf3ZGX4DhPBbISj186O2t/PNvN7kmsGGlxv4gcSSDJh5LM7CSzm6eNy1nTrST6WbOB0OpP1OdS75MSbnuUxFpVEql/az5IakZQH8HXzNnCqu2HXCHcGXpPIb+OkDWo/ZucTwTYSAeVDKUGahVYzHYhYcaxWTWyJZU55IPU1IxjdEhKQy8HfylM9wO5VBIchP9IwIoQqGQayVbrGGJxcxATDLFYhfWDE7ycR8X0xgdksLA28HjK4ZR5qjmKeJeyWd6jEjUsHNHPc8zKDb76MmKFsqabMjHfVxMY3RICgNvB99kZrI+uaYNgFhUpfV0syDH8F7EoaROD1a0UNZky0Dfx8U0RoekMID4Ugab9vQnnsWAT3YcojGLpwgV00XUJEYLZU2xUyxjNCthICLjgOXAdGA78HWlVKfPdjXAY8A0jCjMa5VS27M5diISlTLo7Ol1PW/0jU/38t6W/Vk5bIrlIloMpUgPjUaTX0JZ/v4B4C2l1EzgLfO9H/8G/JNSajZwPhCsklwGJCpaZZl1rHogzryAoYAlBH/6+ibueGw1ja1xMlmj0WgSkq0wWAT80nz9S+AG7wYiciZQopR6A0ApdUQplfnjq1JgTfphTykDy6xz2/wa3+8HO4mEoEaj0QQhW5/BJKWUVWB9DzDJZ5vTgUMi8jwwA3gTeEApFc3y2L44bfmV5WX2pGiZdBpqh87zYJ0UU1SCRqMZfKQUBiLyJjDZ56vvO98opZSI+KUzlwBfBM4F2jB8DHcCv/A51r3AvQA1NZlX77Mm+GT1zIeKELAYSg5t7fvQaPJPSmGglLoi0XcisldEpiildovIFPx9Ae3ABqVUi/mbF4EF+AgDpdQyYBkY5SgCnUEChlJ9/6AMBSFXTBmZGs3JRLY+g5XAN83X3wRe8tlmLTBWRCaY778EfJrlcVNimU1CYNf3147V4kf7PjSawpCtMHgQuFJEtgBXmO8RkXki8hiA6Rv4a+AtEdmIUdPt0SyPmxLLbHLRzPF2SKmeXIqfRAEAGo1mYBnSVUvBMDvctmwVfVFFaVh4+t4LtNmhyNE+A40mc3TV0mSYBeWM/5piZyj4PjSawUa2ZqKix6+gnEaj0WjcDHlhoG3QGo1Gk5ohbyYaSvH3Go1GM1AMeWEA2gat0Wg0qRjyZiLN4KKxtZOH3t6q80E0mjxzUmgGmsGBzj7WaArHSaMZ6BVn8aOzjzWawnFSaAZ6xTk40JVXNZrCcVIIA78VpxYGxYeO/NJoCsdJIQz0inPwoCO/NJrCcFIIA73i1Gg0muScFMIA9IpTo9FoknHSRBNpNBqNJjFaGGg0Go1GCwONRqPRaGGg0Wg0GrQw0Gg0Gg1aGGg0Go2GIn4GsojsB1qz2MV44ECOmpMvdJvzg25zftBtzg/eNtcqpSaku5OiFQbZIiLrMnkodCHRbc4Pus35Qbc5P+SqzdpMpNFoNBotDDQajUYztIXBskI3IAN0m/ODbnN+0G3ODzlp85D1GWg0Go0mOENZM9BoNBpNQLQw0Gg0Gs3gEwYicrWIbBKRrSLygM/3w0Rkufn9GhGZ7vju78zPN4nIl4uozX8pIp+KyB9E5C0RqXV8FxWRDebfyiJq850ist/Rtrsd331TRLaYf98sojb/i6O9m0XkkOO7QvXz4yKyT0SaEnwvIvK/zHP6g4jMdXxXqH5O1eY7zLZuFJGPRORsx3fbzc83iMi6ImrzQhHpcoyBxY7vko6rArb5bxztbTLH8Djzu/T7WSk1aP6AMLANqAPKgE+AMz3b/DnwsPn6VmC5+fpMc/thwAxzP+EiafNlQLn5+j9abTbfHynSfr4T+Fef344DWsz/lebrymJos2f7/wQ8Xsh+No97CTAXaErw/bXAa4AAC4A1hezngG2+0GoLcI3VZvP9dmB8EfbzQuCVbMdVPtvs2fZ64HfZ9PNg0wzOB7YqpVqUUr3AM8AizzaLgF+ar58DLhcRMT9/Ril1Qin1ObDV3F/B26yUelsp1WO+XQ1U56FdyQjSz4n4MvCGUuqgUqoTeAO4eoDa6STdNt8GPJ2HdiVFKfUecDDJJouAf1MGq4GxIjKFwvVzyjYrpT4y2wTFMZ6D9HMisrkXsiLNNmc9ngebMDgF2OF4325+5ruNUioCdAFVAX87EKR73LswVoIWw0VknYisFpEbBqB9fgRt802mOeA5EZmW5m9zTeDjmma4GcDvHB8Xop+DkOi8CtXP6eIdzwp4XUQaReTeArUpEReIyCci8pqI1JufFX0/i0g5xkJghePjtPv5pHns5WBARP4EmAdc6vi4Vim1U0TqgN+JyEal1LbCtNDFy8DTSqkTInIfhjb2pQK3KSi3As8ppaKOz4q1nwctInIZhjC42PHxxWY/TwTeEJE/mivgQrMeYwwcEZFrgReBmYVtUmCuBz5USjm1iLT7ebBpBjuBaY731eZnvtuISAkwBugI+NuBINBxReQK4PvAV5RSJ6zPlVI7zf8twDvAuQPZWJOUbVZKdTja+RjQEPS3A0Q6x70Vj0pdoH4OQqLzKlQ/B0JEvoAxLhYppTqszx39vA94gfyYalOilDqslDpivn4VKBWR8RR5P5skG8/B+zkfjpAcOlRKMBxlM+h35tR7tvk2bgfyr83X9bgdyC3kx4EcpM3nYjipZno+rwSGma/HA1vIg/MqYJunOF5/FVhtvh4HfG62vdJ8Pa4Y2mxudwaGc00K3c+O408nsWPzP+B2IH9cyH4O2OYaDJ/chZ7PRwIVjtcfAVcXSZsnW2MCY+JsM/s80LgqRJvN78dg+BVGZtvPeTmhHHfOtcBmc/L8vvnZUowVNcBw4FlzMH4M1Dl++33zd5uAa4qozW8Ce4EN5t9K8/MLgY3mANwI3FVEbf5HoNls29vAGY7ffsvs/63AnxVLm833S4AHPb8rZD8/DewG+jDs0XcB9wP3m98L8JB5ThuBeUXQz6na/BjQ6RjP68zP68w+/sQcO98vojZ/xzGeV+MQZH7jqhjabG5zJ0ZgjPN3GfWzLkeh0Wg0mkHnM9BoNBrNAKCFgUaj0Wi0MNBoNBqNFgYajUajQQsDjUajKRpSFafz2f7rYhS5bBaRp7I5thYGmqLAUTW0SUSeNVPsC92mhSJyYZb7mCoiz6X5mztF5F/N1/eLyJ9m0wbNoOIJAtaYEpGZwN8BFyml6oHvZnNgLQw0xcIxpdQ5Sqk5QC9GPHVKzCzzgWIhRg5CYLztUUrtUkp9LdMGKKUeVkr9W6a/1wwulE9xOhE5VUR+Y9YZel9EzjC/ugd4SJlFAZWRbZwxWhhoipH3gdNE5HoxnknxexF5U0QmAYjIEhH5lYh8CPxKRKabN8l68+9Cc7uFIvKuiLwkIi0i8qBZa/9js9b7qeZ2E0RkhYisNf8uEuM5GPcD/8XUWL7ot51fe5wnYratyXx9p4g8b97YW0Tkfzi2+zMxnrHwMXCR4/MlIvLX5uvTzH74xDxPq/1/Y7bnDyLy383PRorIv5vbNonILQNypTT5YBnwn5RSDcBfA//b/Px04HQR+dAssJhV1VpdqE5TVJgr62uA3wAfAAuUUkqMh+f8V+CvzE3PxCjGdcw0KV2plDpuqs5PYxT8AzgbmI2x2moBHlNKnS8if4HxTIPvAj8H/kUp9YGI1AC/VUrNFpGHMZ5z8M9m257ybmfu29WeFKd4Dkb5kRPAJhH5/4AI8N8x6jt1YWR0/97nt09iZE+/ICLDgZCIXIVRUO18jGzllSJyCTAB2KWU+g9m28ekaJemCBGRURja6bMiYn08zPxfgnHtF2LUTHpPRM5SSh3K5FhaGGiKhREissF8/T7wC2AWsFyM+v1lGPV3LFY6Jt5S4F9F5BwgirFislirlNoNICLbgNfNzzdiPFQI4ArgTMfNNtq8Cb0k225lAEEA8JZSqstsz6dALUY9pHeUUvvNz5d7zgERqQBOUUq9AKCUOm5+fhVwFf3CYxTGBPE+8FMR+QnGQ1veD9A2TfERAg4ppc7x+a4d48FBfcDnIrIZ49qvzeRAWhhoioVj3gFvrpr/p1JqpYgsxKgrZHHU8fq/YNR2Ohvj5jnu+O6E43XM8T5G//gPYWggzt/hmPQJsN1R78YJcLYnSvb3oAD/qJR6JO4L4xGZ1wI/FpG3lFJLszyWJs8opQ6LyOcicrNS6lkxBtsXlFKfYJTZvg34v2JUWD0dQ/vNCO0z0BQzY+gvF5zsGb9jgN1KqRjwDYxHFabD6xgmIwBMDQOgG6gIsF22rAEuFZEqESkFbvZuoJTqBtrFfPCOGM/6LscwVX3L0lBE5BQRmSgiU4EepdT/A/4J4/GJmiJHRJ4GVgGzRKRdRO4C7gDuEhGr8Jz1pLXfAh2mhvk28DfKUS48XbRmoClmlmDYSjsxnko2I8F2/xtYIUYI5m8Ivkq3+M/AQyLyB4x74j0M5/HLwHMisghDCCTaLiuUUrtFZAnGJHAIo9KnH98AHhGRpRiVLG9WSr0uIrOBVaaGcgT4E+A04J9EJGZu+x+zbadm4FFK3ZbgqzjnsDKqjP6l+Zc1umqpRqPRaLSZSKPRaDRaGGg0Go0GLQw0Go1GgxYGGo1Go0ELA41Go9GghYFGo9Fo0MJAo9FoNMD/D0dZZNjdBFWYAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(torch.arange(0,len(conc)), conc, '.')\n",
+ "plt.title('Parameter Values') \n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"Parameter_Values.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAv/0lEQVR4nO3de5wcVZ338c93JgkQCJCEAQMhJIFwCa4GGCGIKyiCwIKwihpkJSgssouurOuusOsjiPosuiKuNyACAj5cFdGIKETkIgiBDIRLApEhJCQhhJAbIYEkk/k9f9TpSaUz092TTM90Zr7v16teU3XqVNWp6pr+dZ1TdUoRgZmZWSl1PV0AMzOrfQ4WZmZWloOFmZmV5WBhZmZlOViYmVlZDhZmZlaWg0UfIGmkpJDUr4K8Z0p6qDvKVaYcMyQd1dPlsL5F0t9KmtXT5ahFDhY1RtIcSWsl7VKU/mT6wh/ZQ0XLB503c8NT1dhWRBwYEfd3xbokHSWptajcv+2KdW9NJF0saV3a/+WS/iLp8J4uV1dJn/P8Ti4TkvYpTEfEnyNiv64v3dbPwaI2vQScVpiQ9DfAwJ4rziZ2jogd0vDurlxxJVc/m7n8K7ky7xARJ3X1trcSt0bEDsAuwH3AL7p6A8r4u6WX8Qdam34OnJGbngjckM8gaSdJN0haLGmupK8W/kEl1Uv6rqTXJc0G/q6dZa+RtFDSAknflFS/JQWWtLukyZKWSmqW9I+5eddJ+mZueqNfgOlq6iuSngZWSeqX0j6U5tdJukDSi5KWSLpN0pA0r3C1c5akl4E/daLMZ0p6WNLlkpYAF0vaJh27lyUtknSlpO1yy/x7Om6vSPps/peppPslnV20/ody0/tLmpKO0SxJnyg6Rj+W9DtJKyVNlbR3bv6BuWUXSfpPSe+QtFrS0Fy+g9M50b/UvkdEC3AjsIekhrRsh+dF7lj9SNIKSc9LOjq33fslfUvSw8BqYHSZ/T1B0sy0rwskfTk370RJ07Xh6udduXlzJH1Z0tOpHLdK2lbS9sDvgd214epxd0mHSnokrWthKv+AtK4H02qfSvk/2c65eUDat+XKqkY/Uuln1ts4WNSmR4Ed04laD0wA/l9Rnh8COwGjgSPJgstn0rx/BE4EDgIagVOLlr0OaAH2SXmOBc5my9wCzAd2T9v7v5I+2InlTyMLajunL7K8LwCnkO3n7sAy4MdFeY4EDgA+3MlyHwbMBnYDvgVcCuwLjCM7PnsAXwOQdBzwZeAYYAzwoUo3kr7MpgA3AbuSfaY/kTQ2l20C8HVgMNCcyoOkQcAfgT+Q7f8+wL0R8SpwP/CJ3Do+DdwSEevKlGcA2TmzhOx4Qvnz4jDgRbKrkouAXxWCdm7b5wCDgMVl9vca4HMRMQh4JynISzoIuBb4HDAUuAqYLGmb3HY+ARwHjALeBZwZEauA49n4CvIVYD3wr6nMhwNHA/8MEBHvT+t7d8p/a9Ex6g/8Frgn7cMXgBsl5aup2v3MeqWI8FBDAzCH7Evoq8B/k/1TTAH6AQGMBOqBtcDY3HKfA+5P438Czs3NOzYt24/sS3ENsF1u/mnAfWn8TOChDso2Mq1neW74MrAn2T/loFze/wauS+PXAd/MzTsKmF+0z59t7zik8eeAo3PzhgHr0v4UyjS6xDE9CmgtKvcn0r6+nMsnYBWwdy7tcOClNH4tcGlu3r5p2/uk6fuBs3Pz244l8Engz0Xlugq4KHeMrs7NOwF4Pvf5PNnBvn0SeDiN1wOvAod2kPfidN4sT5/XEuCoNK+S8+IVQLn5jwGfzu37JUXlKrW/L5OdszsW5bkC+EZR2izgyNx58Q+5ed8BrmzvvOrgGJwP3JGbbvv8itcB/G06nnW5+TcDF5f7zHrj0BfqaLdWPwceJPv1dEPRvF2A/sDcXNpcsl/BkP36nFc0r2CvtOxCSYW0uqL85ewSuV//kg4DlkbEyqJtNnZinaW2vxdwh6TWXNp6si+4SpaH7Bfn8HyCpDOLlmsgaxtqyh0bkX0JQ3Zcm3L588e1nL2AwyQtz6X1I/ucC17Nja8Gdkjje5L9om/Pb4ArJY0C9gNWRMRjJcpxW0T8g7IbKG4HDiH7oq/kvFgQ6VsxmUt2TAryecvt78fIfhBdqqz68YKIeCQtN1HSF3LLDSjaTvFxys/biKR9ge+RnYsDUxmaOspfZHdgXkTkz7v8/1l7ZdmBXsrBokZFxFxJL5H9WjmraPbrZL+s9wJmprQRwII0vpDsC4bcvIJ5ZL8gN/rC30KvAEMkDcoFjHx5VrFxA/072llHqe6P55FdeTxcPEMb7g7b3O6T88u9DrwFHBgRC9rJW+q4Qun9nAc8EBHHbEYZ55FVd2wiIt6WdBvwD8D+bBx8OhQRr0s6B5gm6SYqOy/2kKRcwBgBTM6vtqjMHe5vRDwOnJyqej4P3EZ2bOcB34qIzanOae8cuAJ4EjgtIlZKOp9Nq2U78gqwp6S6XMAYAfx1M8q21XObRW07C/hgZPWxbSJiPdk/17ckDZK0F/AlNrRr3Ab8i6ThkgYDF+SWXUhWB3uZpB2VNR7vLenIzS1kRMwD/gL8d2psfFcqe6E804ETJA2R9A6yqoDOuJJsX/cCkNQg6eTNLW9H0hfCT4HLJe2atrWHpEI7yG3AmZLGShpIVm+fNx34qKSByhq980H+TmBfSZ+W1D8N75F0QAVFuxMYJul8ZQ3wg9LVXMENZNVEH6HCYJH2dxZwN/AfFZ4Xu5KdV/0lfZysjeiuEmVud38lDZB0uqSdImtbeYOsmhCy43+upMOU2V7S36V2m3IWAUMl7ZRLG5TW/6ak/YF/ameZ0R2sbyrZ1cJ/pPIfBZxE1j7X5zhY1LCIeDEipnUw+wtkv2RnAw+RNSRem+b9lOxL4CngCeBXRcueQXZpP5OscfOXZO0AW+I0svaDV4A7yOqm/5jm/TyVZQ7ZF9Kt7Sxfyv+S/YK9R9JKshsADiu9yGb7CllD5aOS3iBrWN4PICJ+D3yfrE2omU3vvLqcrE1gEXA92d1GpGVXkrUdTSA7Rq8C3wa2oYy07DFkX1SvAi8AH8jNf5jsy/aJiOhM1RjA/wDnpOBY7ryYStaw/zpZQ+6pEbGkRJlL7e+ngTnpGJ8LnJ6Wm0Z2g8aPUhmayQJhWRHxPFmbwux099LuZG1qnwJWkv1fFJ97FwPXp/yfKFrfWrJjfnza558AZ6Tt9DnauArSzDpDUgBjIqK5h8vxJ+CmiLi6Sus/k6zx/n3VWL/VPrdZmG3lJL0HOBjo8qo5swJXQ5ltxSRdT1ZVdn7R3WhmXcrVUGZmVpavLMzMrKxe2Waxyy67xMiRI3u6GGZmW5WmpqbXI6KhvXm9MliMHDmSadM6uuPUzMzaI6nDW69dDWVmZmVVPVgo6y77SUl3pulRqSvfZmXdCxe6C94mTTen+SNz67gwpc/KPU1rZmbdpDuuLL5I1mtowbeByyNiH7InNAtdIpwFLEvpl6d8pC6NJwAHkvXA+hNt4bsXzMysc6oaLCQNJ3tHwdVpWsAHyboRgKxLhFPS+MlpmjT/6JT/ZLL++ddExEtkj/8fWs1ym5nZxqp9ZfF94D/Y0EnYUGB5rlfL+Wzo7ncPUhfHaf6KlL8tvZ1l2kg6R9I0SdMWL17cxbthZta3VS1YSDoReC0iKu07fotExKSIaIyIxoaGdu/8MjOzzVTNW2ePAD4i6QRgW2BHst5Dd5bUL109DGfDOw8WkPVnP19SP7JXhi7JpRfkl+lyTXOX8ejsJYwfPZRD9hpcrc2YmW1VqnZlEREXRsTwiBhJ1kD9p4g4HbiPDS8fmUj2pi/IuqCemMZPTfkjpU9Id0uNIusiudSbwDZb09xlnH71o1x2zyxOv/pRmuYuK7+QmVkf0BPPWXwF+JKkZrI2iWtS+jVkLy5pJnuRzwUAETGD7KUzM8leWH9eevlPl3t09hLWtrTSGrCupZVHZ7fbVb+ZWZ/TLU9wR8T9ZO/5JSJm087dTBHxNvDxDpb/FtnLVqpq/OihDOhXx7qWVvr3q2P86KHV3qSZ2VahV3b3sbkO2WswN5493m0WZmZFHCyKHLLXYAcJM7Mi7hvKzMzKcrAwM7OyHCzMzKwsBwszMyvLwcLMzMpysDAzs7IcLMzMrCwHCzMzK8vBwszMynKwMDOzshwszMysLAcLMzMry8HCzMzKcrAwM7OyqhYsJG0r6TFJT0maIenrKf06SS9Jmp6GcSldkn4gqVnS05IOzq1roqQX0jCxg02amVmVVPN9FmuAD0bEm5L6Aw9J+n2a9+8R8cui/MeTvV97DHAYcAVwmKQhwEVAIxBAk6TJEeEXZJuZdZOqXVlE5s002T8NUWKRk4Eb0nKPAjtLGgZ8GJgSEUtTgJgCHFetcpuZ2aaq2mYhqV7SdOA1si/8qWnWt1JV0+WStklpewDzcovPT2kdpRdv6xxJ0yRNW7x4cVfviplZn1bVYBER6yNiHDAcOFTSO4ELgf2B9wBDgK900bYmRURjRDQ2NDR0xSrNzCzplruhImI5cB9wXEQsTFVNa4CfAYembAuAPXOLDU9pHaWbmVk3qebdUA2Sdk7j2wHHAM+ndggkCTgFeDYtMhk4I90VNR5YERELgbuBYyUNljQYODalmZlZN6nm3VDDgOsl1ZMFpdsi4k5Jf5LUAAiYDpyb8t8FnAA0A6uBzwBExFJJ3wAeT/kuiYilVSy3mZkVUUSpG5S2To2NjTFt2rSeLoaZ2VZFUlNENLY3z09wm5lZWQ4WZmZWloOFmZmV5WBhZmZlOViYmVlZDhZmZlaWg4WZmZXlYGFmZmU5WJiZWVkOFmZmVpaDhZmZlVU2WEjaV9K9kp5N0++S9NXqF83MzGpFJVcWPyV7YdE6gIh4GphQzUKZmVltqSRYDIyIx4rSWqpRGDMzq02VBIvXJe0NBICkU4GFVS2VmZnVlEpefnQeMAnYX9IC4CXgH6paKjMzqyllrywiYnZEfAhoAPaPiPdFxJxyy0naVtJjkp6SNEPS11P6KElTJTVLulXSgJS+TZpuTvNH5tZ1YUqfJenDm7uzZma2ecpeWUj6WtE0ABFxSZlF1wAfjIg3JfUHHpL0e+BLwOURcYukK4GzgCvS32URsY+kCcC3gU9KGkvWoH4gsDvwR0n7RsT6zuyomZltvkraLFblhvXA8cDIcgtF5s002T8NAXwQ+GVKvx44JY2fnKZJ849WFplOBm6JiDUR8RLZO7oPraDcZmbWRcpeWUTEZflpSd8F7q5k5ZLqgSZgH+DHwIvA8ogo3E01H9gjje8BzEvbbJG0Ahia0h/NrTa/TH5b5wDnAIwYMaKS4pmZWYU25wnugcDwSjJGxPqIGJfyHwrsvxnbq0hETIqIxohobGhoqNZmzMz6pEraLJ4h3TYL1JM1dJdrr9hIRCyXdB9wOLCzpH7p6mI4sCBlWwDsCcyX1A/YCViSSy/IL2NmZt2gkiuLE4GT0nAssHtE/KjcQpIaJO2cxrcDjgGeA+4DTk3ZJgK/SeOT0zRp/p8iIlL6hHS31ChgDFD8kKCZmVVRh1cWkoak0ZVFs3aUREQsLbPuYcD1qd2iDrgtIu6UNBO4RdI3gSeBa1L+a4CfS2oGlpK6FImIGZJuA2aSPTl+nu+EMjPrXsp+vLczQ3qJrPpJ7cyOiBhdzYJticbGxpg2bVpPF8PMbKsiqSkiGtub1+GVRUSMql6RzMxsa1JJdx9IGkzWVrBtIS0iHqxWoczMrLZUcjfU2cAXye5Cmg6MBx4he7jOzMz6gEruhvoi8B5gbkR8ADgIWF7NQpmZWW2pJFi8HRFvQ9bZX0Q8D+xX3WKZmVktqaTNYn56XuLXwBRJy4C51SyUmZnVlkr6hvr7NHpxegp7J+APVS2VmZnVlEoauH9A1uvrXyLigW4ok5mZ1ZhK2iyagK9KelHSdyW1+8CGmZn1XpW8Ke/6iDiB7I6oWcC3Jb1Q9ZKZmVnN6EwX5fuQdTG+F/B8dYpjZma1qGywkPSddCVxCfAM0BgRJ1W9ZGZmVjMquXX2ReDwiHi92oUxM7PaVMmts1d1R0HMzKx2bc5rVc3MrI9xsDAzs7IqaeC+TNKBnV2xpD0l3SdppqQZkr6Y0i+WtEDS9DSckFvmQknNkmZJ+nAu/biU1izpgs6WxczMtkwlDdzPAZMk9QN+BtwcESsqWK4F+LeIeELSIKBJ0pQ07/KI+G4+s6SxZK9SPRDYHfijpH3T7B+TvcN7PvC4pMkRMbOCMpiZWReo5KG8qyPiCOAMYCTwtKSbJH2gzHILI+KJNL6SLOjsUWKRk8m6FVkTES8BzcChaWiOiNkRsRa4JeU1M7NuUlGbhaR6sgfy9gdeB54CviTplgqXH0n2HoypKenzkp6WdG16Cx9kgWRebrH5Ka2jdDMz6yaVtFlcTvbE9gnA/42IQyLi2+nBvIMqWH4H4Hbg/Ih4A7gC2BsYBywELtv84m+0nXMkTZM0bfHixV2xSjMzSyq5sngaGBcRn4uIx4rmHVpqQUn9yQLFjRHxK4CIWBQR6yOiFfhpbh0LgD1ziw9PaR2lbyQiJkVEY0Q0NjQ0VLBbZmZWqUqCxXJyDeGSdpZ0CkCphm5JAq4BnouI7+XSh+Wy/T3wbBqfDEyQtI2kUcAY4DHgcWCMpFGSBpA1gk+uoNxmZtZFKrkb6qKIuKMwERHLJV1E9ua8Uo4APg08I2l6SvtP4DRJ44AA5gCfS+udIek2YCbZnVTnRcR6AEmfB+4G6oFrI2JGJTtnZmZdo5Jg0d7VRyXdhDwEqJ1Zd5VY5lvAt9pJv6vUcmZmVl2VVENNk/Q9SXun4XtkL0QyM7M+opJg8QVgLXBrGtYA51WzUGZmVlsqqU5aBbiLDTOzPqxssEhdbnyZ7OnttvwR8cHqFcvMzGpJJQ3cvwCuBK4G1le3OGZmVosqCRYtEXFF1UtiZmY1q5IG7t9K+mdJwyQNKQxVL5mZmdWMSq4sJqa//55LC2B01xfHzMxqUSV3Q43qjoKYmVntqqTX2YGSvippUpoeI+nE6hfNzMxqRSVtFj8jeyjvvWl6AfDNqpXIzMxqTiXBYu+I+A6wDiAiVtN+n09mZtZLVRIs1krajqxRG0l7k3X5YWZmfURFXZQDfwD2lHQjWdfjZ1azUGZmVlsquRtqiqQngPFk1U9fjIjXq14yMzOrGZX0DfX+NLoy/R0riYh4sHrFMjOzWlJJNVT+Ybxtyd6Z3QS4I0Ezsz6ibAN3RJyUG44B3gksK7ecpD0l3SdppqQZkr6Y0odImiLphfR3cEqXpB9Iapb0tKSDc+uamPK/IGliR9s0M7PqqORuqGLzgQMqyNcC/FtEjCVr7zhP0liyd2PcGxFjgHvZ8K6M44ExaTgHuAKy4ELWyH4Y2VXNRYUAY2Zm3aOSNosfkm6bJQsu44Anyi0XEQuBhWl8paTngD2Ak4GjUrbrgfuBr6T0GyIigEcl7SxpWMo7JSKWpvJMAY4Dbq5kB83MbMtV0mYxLTfeAtwcEQ93ZiOSRgIHAVOB3VIgAXgV2C2N7wHMyy02P6V1lF68jXPIrkgYMWJEZ4pnZmZlVHLr7PVbsgFJOwC3A+dHxBvShoe/IyIkRYcLd0JETAImATQ2NnbJOs3MLFNJNdQzbKiG2mgW2ff9u0os258sUNwYEb9KyYskDYuIhama6bWUvgDYM7f48JS2gA3VVoX0+8uV28zMuk4lDdy/J3uC+/Q03JWGE4GTOlpI2SXENcBzEfG93KzJbHhHxkTgN7n0M9JdUeOBFam66m7gWEmDU8P2sSnNzMy6SSVtFsdExEG56QskPRERF3S4ROYI4NPAM5Kmp7T/BC4FbpN0FjAX+ESadxdwAtAMrAY+AxARSyV9A3g85buk0NhtZmbdo5JgIUlHFBq1Jb2Xyp7PeIiOe6c9up38AZzXwbquBa6toKxmZlYFlQSLs4BrJe2UppcDn61aiczMrOZUcjdUE/DuQrCIiBVVL5WZmdWUSl6rupuka4BbImKFpLGpvcHMzPqISu6Guo7s7qPd0/RfgfOrVB4zM6tBlQSLXSLiNqAVICJagPVVLZWZmdWUSoLFKklD2fBa1fGA2y3MzPqQSu6G+hLZA3N7S3oYaABOrWqpzMysppQMFpLqgSPTsB/ZcxOzImJdN5TNzMxqRMlqqIhYD5wWES0RMSMinnWgMDPreyqphnpY0o+AW4FVhcSIKPtOCzMz6x0qCRbj0t9LcmmB38FtZtZndBgsJH0xIv4X+D+pnyczM+ujSrVZfCb9/UF3FMTMzGpXqWqo5yS9AOwu6elcetmXHpmZWe/SYbCIiNMkvYOsq4+PdF+RzMys1pRs4I6IV4F3d1NZzMysRlXS3cdmkXStpNckPZtLu1jSAknT03BCbt6FkpolzZL04Vz6cSmtWVK5t/OZmVkVVC1YkPVWe1w76ZdHxLg03AUgaSwwATgwLfMTSfXpCfIfA8cDY4HTUl4zM+tGFQcLSQM7s+KIeBCo9F3ZJ5O9L2NNRLxE9h7uQ9PQHBGzI2ItcEvKa2Zm3aiSlx+9V9JM4Pk0/W5JP9mCbX5e0tOpmmpwStsDmJfLMz+ldZTeXjnPkTRN0rTFixdvQfHMzKxYJVcWlwMfBpYARMRTwPs3c3tXAHuTPRW+ELhsM9eziYiYFBGNEdHY0NDQVas1MzMq6+6DiJgnKZ+0WS8/iohFhXFJPwXuTJMLgD1zWYenNEqkm5lZN6nkymKepPcCIam/pC8Dz23OxiQNy03+PVC4U2oyMEHSNpJGAWOAx4DHgTGSRkkaQNYIPnlztm1mZpuvkiuLc4H/JWsrWADcA/xzuYUk3QwcBewiaT5wEXCUpHFkHRHOAT4HEBEzJN0GzARagPNS9+hI+jzZg4H1wLURMaPy3TMzs66giCidQToiIh4ul1ZLGhsbY9q0aT1dDDOzrYqkpohobG9eJdVQP6wwzczMeqlSXZQfDrwXaJD0pdysHcmqhMzMrI8o1WYxANgh5RmUS38DOLWahTIzs9pSqtfZB4AHJF0XEXO7sUxmZlZjKrkb6jpJm7SCR4Rfq2pm1kdUEiy+nBvfFvgY2e2tZmbWR5QNFhHRVJT0sKTHqlQeMzOrQWWDhaQhuck64BBgp6qVyMzMak4l1VBNZE9ci6z66SXgrGoWyszMaksl1VCjuqMgZmZWu0o9lPfRUgtGxK+6vjhmZlaLSl1ZnFRiXgAOFmZmfUSph/I+050FMTOz2lXJa1V3kvS9witLJV0myXdDmZn1IZX0OnstsBL4RBreAH5WzUKZmVltqeTW2b0j4mO56a9Lml6l8piZWQ2q5MriLUnvK0xIOgJ4q9xCkq6V9JqkZ3NpQyRNkfRC+js4pUvSDyQ1S3pa0sG5ZSam/C9Imti53TMzs65QSbD4J+DHkuZImgv8iOxVq+VcBxxXlHYBcG9EjAHuTdMAx5O9d3sMcA5wBbQ9PX4RcBhwKHBRIcCYmVn3qeShvOnAuyXtmKbfqGTFEfGgpJFFySeTvZcb4HrgfuArKf2GyN7x+qiknSUNS3mnRMRSAElTyALQzZWUwczMukYld0N9MQWKlcD3JD0h6djN3N5uEbEwjb8K7JbG9wDm5fLNT2kdpbdXznMKd2wtXrx4M4tnZmbtqaQa6rPpauJYYCjwaeDSLd1wuorY5D0ZW7C+SRHRGBGNDQ0NXbVaMzOjsmCh9PcEsqqiGbm0zlqUqpdIf19L6QuAPXP5hqe0jtLNzKwbVRIsmiTdQxYs7pY0CGjdzO1NBgp3NE0EfpNLPyPdFTUeWJGqq+4GjpU0ODVsH5vSzMysG1XynMVZwDhgdkSsljQUKNsViKSbyRqod5E0n+yupkuB2ySdBcwle8gP4C6yYNQMrC6sPyKWSvoG8HjKd0mhsdvMzLqPsqaDMpmyHmjfR9bG8FBE3FHtgm2JxsbGmDZt2mYt2zR3GY/OXsL40UM5ZC/fpWtmfYekpohobG9eJW/K+wmwDxtuV/2cpA9FxHldWMaa0DR3Gadf/ShrW1oZ0K+OG88e74BhZkZl1VAfBA5Idy8h6XpgZlVL1UMenb2EtS2ttAasa2nl0dlLHCzMzKisgbsZGJGb3hN4oTrF6VnjRw9lQL866gX9+9UxfvTQni6SmVlNKPWmvN+StVEMAp6T9FiaPgx4rHuK170O2WswN5493m0WZmZFSlVDfbfEvC57mK7WHLLXYAcJM7Mipd6U90B76akH2tOAB6tVKDMzqy2VNHAj6SDgU8DHgZeA26tZqJ7kW2fNzDZVqs1iX7IriNOA14FbyZ7L+EA3la3b+dZZM7P2lbob6nmy22ZPjIj3RcQPgfXdU6yekb91ds26Vr7/x7/SNHdZTxfLzKzHlaqG+igwAbhP0h+AW9j8DgS3CoMHDqA1Nd0H8NALrzN19hI+3rgnHz14eLtXGa62MrO+oMMri4j4dURMAPYH7gPOB3aVdMUWvM+ipj37yoqNpgNYuz64aerLnH71o5tcZRSqrS67Z1a7883MeouyD+VFxKqIuCkiTiLrIvxJsrfb9TodXTYFG57ozmvviW8zs96okie420TEsvSSoaOrVaCedODuO200fejIwSWf6PYT32bWV1R062xfsWz12rZxAUfutytfOf6ADtsk/MS3mfUVDhY5gwcOaBuPoumO+IlvM+sLHCxyihu475v1GpfcOcPPXZhZn9epNouuImmOpGckTZc0LaUNkTRF0gvp7+CULkk/kNQs6WlJB1etXEXTz8xf7gZsMzN6KFgkH4iIcbm3Ml0A3BsRY4B70zTA8cCYNJwDXFGtAn304OEM6LfhkCx6Yw2tAXVuwDazPq4ng0Wxk4Hr0/j1wCm59Bsi8yiws6Rh1SjAIXsN5uKTDmTk0IGIrN2iDjhin13aqqCa5i7jx/c1+5kKM+tTeqrNIoB7JAVwVURMAnaLiIVp/qvAbml8D2Bebtn5KW0hXaxp7jIuuXMGa9a1ZoFCMKBfHed/aN+2QOG+o8ysL+qpYPG+iFggaVdgiqTn8zMjIlIgqZikc8iqqRgxYkSZ3O17dPYS3l7X2jY9umEHvv2xd7UFBL921cz6qh6phoqIBenva8AdwKHAokL1Uvr7Wsq+gOxVrgXDU1rxOidFRGNENDY0NGxWuV5YtHKj6ebX3uSqB15sq3LyQ3hm1ld1e7CQtL2kQYVx4FjgWWAyMDFlmwj8Jo1PBs5Id0WNB1bkqqu61PR5yzdJu2fmorZ+nwoP4X3p2P1cBdUN3D5kVjt6ohpqN+AOSYXt3xQRf5D0OHCbpLOAucAnUv67gBOAZmA18JlqFWzEkIHMWbJ6k/S317Vy3o1N/MvR+/Kpw0Y4SHQDtw+Z1ZZuDxYRMRt4dzvpS4BN+pyKiADO64aiseKtdR3Oe/WNNfznHc/w8pJVDNquv7v3qDK3D5nVFj/BnbPrjtsCK0rmuerB2QDU14lLTn4n+71jkPuGqoJC+9C6lla3D5nVAAeLnA/stytTZi4qmadwi1ZLa/B/fv0M9fV1tKx3VUlXcyeNZrXFwSJn2eq1bQ/jVaI1oLWldaP3XfhLreu4k8bK+Y2NVm0OFjmdreqoqwMQEeGqkirwF2BlfDOAdQcHi5yfPzKn4qsKgPWtAEGd4MzDR/oftIzOfPn7C7ByvhnAukMt9Q3V4373TOnHN+oE++y6wybprQFX/Xl2xc8D9IbnBy696zmO+p/7uPSu5yrK3zR3GRMmPcL/3D2LCZMeKbvvfmVt5fywqHUHX1nkrFtf+rqiNeDF195sd14EfPv3z3Hbue8tuY6mucv45KRHaFkf9KsXt55z+Fb3K/DSu57jynRXWOHvBSccUHKZqx54se34rlsffOX2pzfqSqVY/gsv6HwVYV/imwGsO/jKopNKhZNpFVwpXPXAi7SkL82W9cFVD7zYRSXrPjc99nLJ6fZMnbPxlUHza2/yyase4aapG5YtXHHdNPVlvnrHM7Smg90aWRVhb7gi6yo+FtbdfGXRhVrbiSTF9fSPzVm60fxFb7zdTaXbfMX7sHpty0bzC9M3TX2Z3z+7kOPfOWyT509WvtWyyXpbWoP/Sg86jhi6PV/7zbOsb412A/Kvp7/CXc++6tuU2bQ952snHsjFv53R9kzKzf84HqDt+OfH++oxsy3nYNHF9r7wdzQM2obxo4eyYPlbPD4n++VXLxi9y/YsX73xU+JL3lzTNt40dxlXPvAir73xNp98zwg+ddjm9Z67Oc6/5Unu/+tijtq3ge9POGijMhU3NLe0brxsSyuccc1UHnzhdQD+nP4WDNqmvt1ACtmVWqEqq5y1acNrc20Y+S/Bnjx+1ZYP2L96Yn5bN/rrWlq59fGXNzo2X/nlU7y87C1a1rfSr04gOcjaFnOw6GLrI+sa5NfTX9kk/YXFqzbJP3/524z7+t1sv21/Fi5/q+1L9an5zwDwqcNGlLyLqHhe4QvzpcVvMrphBz535N5lvxzOv+XJtvIW/n5/wkHZ+z1+O6Ot2/ZCH1ntebAoQOStXLO+5PY7qzWyHoIv/+NfaVkf1NfBISMG8/icZW1XJU/N77hrlqa5y7j9ifkIOHD3nVi2em2X/+rOfy5Q/pd9R/kBbn9iPrdNm8f6tK9IbftZX1/XFigKmnPn2dr1QaHydM26Vm5/Yn7beeKrDesMZV0v9S6NjY0xbdq0Ti838oLfVaE0m69/vRg6cACvrtxw9XHu+0dzwQkHtH3h/bJpftuvxq+deCBf/fUzG/2Kr0tXNKMbduCo/Xbl2VdW8KeZi3h91RqGDx7IZZ8Yx4RJj2zSuN+ZhxO3BgK2H1CP6sTKtzetEgMY07A9U/7tqLbpwhfq4IEDuG/WayWvWIq/7D92xV/a5tUpC3D1gm+c8jcbBafzb3mSu55ZyNr1gVLe/EdR7nMYtE19p4PxXkMGMnfphg4zjx27G587cm+AtiD60YOHl/1h0luU26/C/JVvrWPGwjc4/p3DetVVa56kptyrrjee52CxQa0FC+sZew0ZyGsr3+btVNVTbMjA/uw5ZCCffM8IXl6yil9PX8CiN9Z0OrgOqFf65V+bDh2ZfXHOfn0Vq9eu5+1164n0TvqjD9gQYAoBtdQVWiWBpqM8+aBdyTYGDxzAs6+soHnRSta0tDJql+156fVVrG1p5eWlq1nT0so7d9+RYw58By8sWsnkp15pC+b77jaIV1a8xao16xmyfX8+etBwrntkzkYvRQOor4P9dhvEN075m7ZjkL+6Lw66lQakzgbirg7gDhYVcrAw23JDBvbnjbfXUSdRL7GuNWhJl7v968Xho4fy+JylbcF45+36EcCK3E0Q575/NI/MXsK8patZunrT3qAPHTmYBcvfYsmqtZt8kdea4h8F2/WvY0D/Ota1BHWw0dWugPekfUPiwGE7tlUlF4JQ86KVLF21lv71dfz1tTdpbQ3q68TZ7xvFG2taOrwyrISDRYUcLMyst7j9n97b6YBRKlj4OQszs17oU5Me6dL1OViYmfVCa7q4PWyrCRaSjpM0S1KzpAt6ujxmZn3JVhEsJNUDPwaOB8YCp0ka27OlMjPrO7aKYAEcCjRHxOyIWAvcApzcw2UyM+sztpZgsQcwLzc9P6W1kXSOpGmSpi1evLhbC2dm1tttLcGirIiYFBGNEdHY0NDQ08UxM+tVtpZgsQDYMzc9PKV1qTmX/l1Xr9LMrEd09ffZ1tKR4OPAGEmjyILEBOBT1diQA4aZ2aa2imARES2SPg/cDdQD10bEjB4ulplZn7FVBAuAiLgLuKuny2Fm1hdtLW0WZmbWgxwszMysLAcLMzMry8HCzMzK6pXvs5C0GJi7BavYBej4pdLm41Oaj09pPj6l9eTx2Ssi2n2quVcGiy0laVpHLwAxH59yfHxK8/EprVaPj6uhzMysLAcLMzMry8GifZN6ugA1zsenNB+f0nx8SqvJ4+M2CzMzK8tXFmZmVpaDhZmZleVgkSPpOEmzJDVLuqCny9PVJO0p6T5JMyXNkPTFlD5E0hRJL6S/g1O6JP0gHY+nJR2cW9fElP8FSRNz6YdIeiYt8wNJKrWNWiOpXtKTku5M06MkTU37c6ukASl9mzTdnOaPzK3jwpQ+S9KHc+ntnl8dbaMWSdpZ0i8lPS/pOUmH+/zZQNK/pv+tZyXdLGnbXnMORYSHrN2mHngRGA0MAJ4CxvZ0ubp4H4cBB6fxQcBfgbHAd4ALUvoFwLfT+AnA7wEB44GpKX0IMDv9HZzGB6d5j6W8Sssen9Lb3UatDcCXgJuAO9P0bcCENH4l8E9p/J+BK9P4BODWND42nTvbAKPSOVVf6vzqaBu1OADXA2en8QHAzj5/2o7NHsBLwHa5z/XM3nIO9fgBrpUBOBy4Ozd9IXBhT5eryvv8G+AYYBYwLKUNA2al8auA03L5Z6X5pwFX5dKvSmnDgOdz6W35OtpGLQ1kb2C8F/ggcGf6wnod6Fd8jpC9W+XwNN4v5VPxeVPI19H5VWobtTYAO6UvQxWl+/yJtmAxjywI9kvn0Id7yznkaqgNCh90wfyU1iulS96DgKnAbhGxMM16FdgtjXd0TEqlz28nnRLbqCXfB/4DaE3TQ4HlEdGSpvP703YM0vwVKX9nj1mpbdSaUcBi4Gepqu5qSdvj8weAiFgAfBd4GVhIdk400UvOIQeLPkjSDsDtwPkR8UZ+XmQ/Tap6P3V3bKOzJJ0IvBYRTT1dlhrWDzgYuCIiDgJWkVUJtemr5w9Aakc5mSyo7g5sDxzXo4XqQg4WGywA9sxND09pvYqk/mSB4saI+FVKXiRpWJo/DHgtpXd0TEqlD28nvdQ2asURwEckzQFuIauK+l9gZ0mFN0rm96ftGKT5OwFL6PwxW1JiG7VmPjA/Iqam6V+SBQ+fP5kPAS9FxOKIWAf8iuy86hXnkIPFBo8DY9JdBQPIGpwm93CZulS6s+Qa4LmI+F5u1mSgcEfKRLK2jEL6GemulvHAilQVcDdwrKTB6dfUsWR1pAuBNySNT9s6o2hd7W2jJkTEhRExPCJGkn32f4qI04H7gFNTtuJjU9ifU1P+SOkT0p0uo4AxZI227Z5faZmOtlFTIuJVYJ6k/VLS0cBMfP4UvAyMlzQwlb9wfHrHOdTTjUK1NJDdvfFXsjsO/quny1OF/Xsf2eX708D0NJxAVud5L/AC8EdgSMov4MfpeDwDNObW9VmgOQ2fyaU3As+mZX7Ehl4C2t1GLQ7AUWy4G2o02T9qM/ALYJuUvm2abk7zR+eW/6+0/7NId/OUOr862kYtDsA4YFo6h35NdjeTz58N5f868Hzah5+T3dHUK84hd/dhZmZluRrKzMzKcrAwM7OyHCzMzKwsBwszMyvLwcLMzMpysLCtnqT1kqannj5/IWlgDZTpKEnv3cJ17C7pl51c5kxJP0rj50o6Y0vKYFbgYGG9wVsRMS4i3gmsBc6tZKHcE6/VcBTQqWBRXJ6IeCUiTu0ofzkRcWVE3LC5y5vlOVhYb/NnYB9JJ6X+/Z+U9EdJuwFIuljSzyU9DPxc0khJf5b0RBrem/IdJekBSb+RNFvSpZJOl/SYsvct7J3yNUi6XdLjaTgiddJ4LvCv6Yrnb9vL11558juSyvZsGj9T0q8k/UHZOx2+k8v3GUl/lfQYWfcS5Nb95TS+TzoOT6X9LJT/31N5npb09ZS2vaTfpbzPSvpkVT4p26pU85eVWbdKv8yPB/4APASMj4iQdDZZb7L/lrKOBd4XEW+lKqtjIuJtSWOAm8meIgZ4N3AAsJTsnQtXR8Shyl4a9QXgfLL+oy6PiIckjSDrtuIASVcCb0bEd1PZbirOl9a9UXnK7OI4sp6C1wCzJP0QaCF7avgQsl5L7wOebGfZG4FLI+IOSdsCdZKOJetK4lCyp60nS3o/0AC8EhF/l8q+U5lyWR/gYGG9wXaSpqfxP5P1f7UfcKuyTucGkL2HoWBy7ou5P/AjSeOA9cC+uXyPR+oWW9KLwD0p/RngA2n8Q8DYrCsgAHZU1qtvsVL5JlcQKADujYgVqTwzgb2AXYD7I2JxSr+1aB+QNAjYIyLuAIiIt1P6sWT9MhWCyw5kwePPwGWSvk3W7cmfKyib9XIOFtYbvBUR4/IJ6Vf39yJisqSjgItzs1flxv8VWER2FVEHvJ2btyY33pqbbmXD/04d2RVMfjlyQYEK8q0qztyBfHnWs+X/vwL+OyKu2mRG9grUE4BvSro3Ii7Zwm3ZVs5tFtZb7cSGbponlsm3MCJagU+TvbqyM+4hq5ICIF2hAKwke3VtuXxbaipwpKShyrqf/3hxhohYCcyXdEra9jap+u1u4LOFKxxJe0jaVdLuwOqI+H/A/5B1Q259nIOF9VYXA7+Q1ET2ysmO/ASYKOkpYH8q/5Vf8C9AY2ognsmGO7F+C/x9oYG7RL4tkqrJLgYeAR4Gnusg66eBf5H0NPAX4B0RcQ/Z+8YfkfQM2fspBgF/AzyWqvYuAr7ZFWW1rZt7nTUzs7J8ZWFmZmU5WJiZWVkOFmZmVpaDhZmZleVgYWZmZTlYmJlZWQ4WZmZW1v8HANQQ3VaCT28AAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(torch.arange(0,len(flat_fft)), flat_fft.abs(), '.')\n",
+ "plt.title('Model Fourier Frequency Representation') \n",
+ "plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"Parameter_Frequency.png\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Normalizing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(-0.0023)"
+ ]
+ },
+ "execution_count": 56,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.mean(conc)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "conc2 = conc #+ 0.0016"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(-0.0023)"
+ ]
+ },
+ "execution_count": 58,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.mean(conc2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 59,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat_fft2 = fft.rfft(conc)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 60,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[<matplotlib.lines.Line2D at 0x7ff775a99c10>]"
+ ]
+ },
+ "execution_count": 60,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEFCAYAAAABjYvXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABYNklEQVR4nO29e3QU153v+/1VtwQIhBDijZBAGGMQjjHCRtiOH7Gd2Ll2cEwc/JiZ+MTPM8m5kztnZo1PMuGySO4c587kjues43tt7Hg5k2NsYmNj7LGTGL8fgEEEjIQtHgIJ8UY0QiBQq7v2/aNql6p2V3VXv1ut32ctFuru6q5du/bev/17FgkhwDAMwwxttHw3gGEYhsk/LAwYhmEYFgYMwzAMCwOGYRgGLAwYhmEYAMF8N8CLcePGienTp+e7GQzDMIOKpqamk0KI8cl+r2CFwfTp07F169Z8N4NhGGZQQUTtqXyPzUQMwzAMCwOGYRiGhQHDMAwDFgYMwzAMWBgwDMMwYGHAMAzDoIiFQVN7CE++vxdN7aF8N4VhGKbgKdg8g3Roag/hvmc3IRzRURrU8MKDjWiorcx3sxiGYQqWotQMNrV1IRzRoQugP6JjU1tXvpvEMAxT0BSlMGisq0JpUEOAgJKghsa6qnw3iWEYpqApSjNRQ20lXniwEZvautBYV8UmIoZhmAQUpTAADIHAQoBhGMYfRWkmYhiGYZKDhQHDMAzDwoBhGIYpcmHAiWcMwzD+KFoHMieeMQzD+KdoNQNOPGMYhvFP0QoDTjxjGIbxT9GaieIlnjW1hzghjWEYxkZGhAER3QLg3wAEADwrhHjc5ZjvA1gBQADYIYS4NxPnjodb4hn7EhiGYWJJ20xERAEATwK4FcBcAPcQ0VzlmFkA/huAq4UQ9QB+ku55U4V9CQzDMLFkQjO4EsBeIUQbABDRSwCWANhlO+YhAE8KIUIAIIQ4noHzxqWpPYS12zpBAO5cUG3t/qUvoT+isy+BYRjGJBPCYCqAg7bXnQAWKcdcDABE9CkMU9IKIcQf1B8ioocBPAwANTU1KTeoqT2Ee1ZtRDgqAAAvN3XixYcaLbMRF7FjGIZxkisHchDALADXA6gG8BERXSqEOG0/SAixCsAqAFi4cKFI9WSb2rrQHx34ujQHyYWfi9gxDMM4yURo6SEA02yvq8337HQCWC+E6BdC7AewG4ZwyAqNdVUIBMh6zeYghmGY+GRCGGwBMIuIZhBRKYC7AaxXjlkHQysAEY2DYTZqy8C5PZEXRgB+eNV0NNRWcnkKhmEYD9I2EwkhIkT0YwB/hOEPeE4I0UJEKwFsFUKsNz/7JhHtAhAF8PdCiKyF8djNRALAqo8NufP8xgPo69cR0Agrl8zDvYtS90swDMMUExnxGQgh3gLwlvLectvfAsDfmv+yTmVZKYgAYboNdGEIBCEM4RDRBf5x3U4AYIHAMAyDIixH0dQewso3W6Ar7mchABpwI0AXwPLXm9lkxDAMgyIUBjKpTKUkqOHhr9dBcwgEwUlnDMMwKEJhIJPKbGs+CMD3Gqrx2Lfn4Jd3XIoAGe8FNeIoowzBznmGGdwUnTCQSWU3zZ1ovScAzJtSgab2EJoPd0OT6oHdbsSkjKz39Os/teK+ZzexQGCYQUjRCQPJ/pPnHK8/aD2O+57dhBc3d6A/KiAARKNcmygTcL0nhhn8FF0J66b2EJat2ohI1OlBPnbmAsIRHfJdAiejZQqu98Qwg5+iEwavbuuMEQQEYHFdFVqP9aA/YuQZ3LVwmqOAHZM6XO+JYQY/RScMvAoaPb/xAO5fPB0tR87g1nmTOb8gw3C9J4YZ3BSdMJg3pSLmPQHgQr9uZSJvbutCy+Fu1gwYhmFMis6BHOoNe36mC+NfOCrwwuYO3PMMR74wDMMARSgMGuuqEPAZMRqO6Hh1W2d2G8QwDDMIKDph0FBbiYe+Xuf5uSonUn5oAsMwTBFRdMKgqT2EZz52r45NBDxybR1KAwQCUBogLF1QndsGMgzDFCBF50Beu60TUY/tPgngTF8E31s4LebZyAzDMEOZohMG8dwFOoCXtx5EVBcoDWq4s8i0gqb2EMf6DxH4XuePYu37ohMGdy6oxktbOhCNLVwKANZDb/r6dazd1pnWzSykQSHrA4UjOkqDGl54sDHvbRpMFNK9TATf6/xRzH1fdD4DYOChNnGPgaElpBpaWmjF2bg+UOo0tYdwzzOb8C9/bB0U4cZ8r/NHMfd90QmDTW1dMQ+28SISTf15BoU2KGR9oADlr+bSYC1j/eq2Tqtu1WAINy6Eez1UKea+LzozUWVZaVLHp3ozC604W77rAw1m9VndO2Qy3Dgb5if1XgPAk+/vTekcg8k8Vgjke55lk4wIAyK6BcC/AQgAeFYI8bjHcUsBvALgCiHE1kycW6X5cLfvY9OZ9IU4KPJZH8hNUyqEPvHD0gXVeGXrQfRHBUoyGG6cTQEp73U65xjMAjyfFGsdrrSFAREFADwJ4GYAnQC2ENF6IcQu5bhyAH8DYHO654zHyZ6+pI5Px4lcaIMin7u8QtOUkqGhthIvPrw4432XCwGZzjkGswBnMk8mNIMrAewVQrQBABG9BGAJgF3Kcb8A8CsAf5+Bc3oyrnxYUsfvPdaTpZbklnzv8gpRU0qGbAj2XAjIdM4xmAU4k3kyIQymAjhoe90JYJH9ACJaAGCaEOI/iMhTGBDRwwAeBoCamtRKTC9dUI01n3d4Jp6p9EU8YlALFK/dfyHs8gpNU0qGXNj2s9E36Zwj1e+yn8E/g6mvsu5AJiINwP8D4P5ExwohVgFYBQALFy5MyaTfUFuJGeNGYu+Jc4kPBlAxoiSV0wDI/Y2Ot/vnXV7q5MK2n03SOUey321qD+GeVRst/8qLDy8u+EXOjVzM3Xxr68mSCWFwCMA02+tq8z1JOYB5AD4g4wH0kwCsJ6LvZMuJPHZkKeBTGHSc6k3pHPm40fF2//k20wymHZBKIWhVuSSde7V2WyfCptodjoq0EzeTQba7sqwUod5wymMtV3N3sI2rTAiDLQBmEdEMGELgbgD3yg+FEN0AxsnXRPQBgL/LliAAgJ4LEd/H3lI/KaVzbGrrQl+/GZven5sbnWj3ny8zjTq5lt9Wn9ZkzTWDQatKZQFXv9PUHsLabZ14pakTkWhqC6Fa7qXlUDea2kNZv89yjMk5RwACGmHlknlJP7UwV4v0YBhXdtIWBkKICBH9GMAfYYSWPieEaCGilQC2CiHWp3uOZHj8rS/x5VF/TuERQQ2PfXtOSuepLCu1QlN1+M9vSGdXlmj3n6/duX1yhft1LH+9GboQBaEa++mTQteq3HaygLFL9yq46CagV77ZYi2mgPtCmKgtdy6oxstNRpIeAOw81I37nt2U9fssx5hsuwAQ0QWWv96M2ZPKkzp3rhbpfI+rZMmIz0AI8RaAt5T3lnsce30mzunFbz5xL1/txtgkI4/sqPkMfvIbMqGeeu3+82mftE8uIkJUF1Y2bz5V42T6JBtalR9B5KeNqha6dlsnXtl60DLXvNzUiRcfcn5P3f2+3XzEsZgSYjNo/bSlobYSLz7UiJVvtGBHZ3fOTCByjIX7ddhDPnQxUEXA76KbiUXa78ZrMAVVFF0Gcn8SwUFHTp9PWcVV1WU/D1fLpnqaT/uknFxrt3Vi77EefH7AKEehi+QzwlMhUxFWmdSs/AoiP21UtdCTPX1WwUXA/Xvq7vfWeZOx5cAp9Ed0BDTCXQunxWgUyfTXl0fOWH8HAtk3gdgX8J7z/Xj2k/2W9llZVmqZkPyajtJZpAebY9gvRScMkkEXqSedSXVZTjY/5bCzqZ7afzugEQ6nIehS5dVtneizSWMN8Z9JnQlkkTnZp/YdcmNdFYIaoT8qENAobn9neoL7XVj9jIlQbxgaGeNVIyOXpiRAlmZQYi6I9pIUbrvf2ZPK4wo7v+NzU1sXImYBMALwvYbcPBfEvoDfXD/Juha75hTRBX6+bmfSpqNkyNbGK99BGEUnDOSk8UuqSWdSXU7m5mXKhug2aOy785e3HsTqzR2u5oNsodp0ASAYMBbgTA9y++/JInPAQJE5+zmiwrAvJ8o7yURAgL1dfhdWP2OisqwUGhEAYye8dEE1li6otnwG9VMqsGJ9M8JRgQABv7jjUty7qCZm95toN9xQW4nlt9Xj7eYjuHXeZM9j1WvLxdMCpfNb+kjU9mg0cI+jLpu8TI7BRPc2VWd/vrWNohMGyQgCADjV2+94vXpzhzUZklE1cyXV4w2ahtpKvLqt0zIhuC2O6Zw30YIV0/dEaD3ag5VvtmRskKvXf+2s8Y7P7U14+sN9iJqNiuoCT3+4D6v+aqHr76YaEODVrhcebPQt+OMt0k3tIax8swVRXUAjWNdr/85PX9tpaQlRAfw8Baeq/VzhiI4tB055/kaqm5pU54jMbZDXuGbrQWhEjoioG+dMxJ92HbO+Q8r3M73Q3rmg2iGYpFYGIOlzNbWH8MSG3XkPQy06YaABSCaneMa4kdbfqzd34Kev7QQAfLznJAD4CluLZ6pQj0t3UCZSUb0qcKYjrPy0u8XFgR6NDjguMzXI1esfVz4MpQFyLTJ37MwFx3fV1/bre7v5CIy9t6FdJmvecrsvP7rhorQntF3jigrgnV3H8NGeE47wXbUel66LlPo5GfNHsjb3dMb+prYuh48kEhUgGEEKsp2PXDcTH+w+4Wq2zaRZR72O+ikVjs3O0gXVSfup7CGzGuWvNHbRCQNrRvvkhtkTrL/fbj7i+Ozt5iMxwsBtUU1kqpD4GZSJFu1EKurSBdX4/ZYORHQgqBmv0xVCidrd1B7Cy1sPOr5DMByLdsdlJga5m4li6YJq1z5bdkUNdnTudLxWcZuMpQna6Ra/f+j0eQQ0goiKlB2qbvdeXq9sn4DxlD4ZvhvUKGbzQ5Sa4z6bPq10FuTGuiqHjyQYIGhEiEYH2hnPbJvMdXnNP/n+4dPnccH0i/X1x252BJBUH9qFvQbg6ovG4Sc3Xcw+g0yQrJno/1w/oFLfOm+ypREAwK3zJjuOtaurGoCF0ysxa2I5Tig7M68m+LE1uiVvqRmXiVR0TdNAug5NM55dlO7OKFG77Q5FZ0cIzJ5UntFYa6/rd/vdexfVoKPrHP7QchS31E+yBLt9wqu+jkunVmD57fVxzTZ2LXDF7c74fTKvOx5upkgvgS2v96kP9+Ed0wwiACt8tz8qYsabLoAVb7R4mnm8FrxEYysd7TIdQdNQa1SVVX0G8rWf7/sZg173wP6+/XwCQNXI0pjNybwpFdb9BeI/a0Ltl3wJAqAIhUGy9EcHVGo5Mb18BvZUfB3A5wdC+PxACMEAIRggRBVThTp57IOysqzUio+WN98teUtOerljtS8Sduw7l0jUjKyI6tb5gwEz0iiFXavdsVg1shRPbNiN+smjUT6ixOEstceAy8iOVM0lqsMwGUeo/Tee33gA4YiO5zcewM1mtvmypz+zNKeVSy6FphF08762HDljPenM7RyqFrhmS0dMMlQ0jplGNUV2dJ3DY9+eY0Vi2U0fdkE3f9oYvPvlMehiIPtWjg034mmebiZN+1j90Q0XufZlOjWJ0g2ekPfcXpJC3ou12zqtpDov7VcdM/brBWDNHbdNk31eqnSdCzuuq/VoD36+bid0AWzcdxKapsXN9s5UUEkmGPLCAPD/tDOvXUg0KnDPohpMHTPCuqGrN3e4ZuLKm+22A1GTt3QxMNkTmZbk79mjqRxx/nK36ucB0QrSsXjBFjb68Z6TIADDSgacpU9s2I1P9py02qxR/HBO++/bJ8PqzR34+bqdVnRIqlFRbhFC2w+ehrmWI6IDr/25E8I2yyNRgRc2d2DNloOu8erHFS2wNKg5BKGbzdd+faopctXHbaipGomXtx60+s1NYKs7yGljRmBPnPpbbuGmgLtJE0js9FRrEv3DKztQN34UxpcPc82AdiNZP4MdtZSGZs4PNanOj/Zrny9BjQDTGR3UCMGA5jA/Ac6EN2mqk9RPHu0QVPZxG9EB6EZf29vktknMpxCQsDAArAnRerQnrgP5zgXVWLP1ICJKnKJUD+2RRfZBoWbieplt5C5h7bZOnOzpwwe7TyAS8V5kJPbfs6/1Ms5fmnES7VrtuJlSVOy72B/dcBF+ctPF2HLgFML9OjQz+Ucu7l7alirIpleNRNvJc45dWKpOP7cIoeOKE/ngqV7XHV9EF/jHdcZYuHdRjdUf6obg4onleOzWOdZuVa3JpJoe7l883WGKFMLQRBPF7dvNRX/uCHkKgitN06Xq2JQLvCrMjvf0+TIjqte998Q5qzLwmq0HsSaL1UtlH9o3I7oQIBqYF9I3Fe43NlLxfCaO640KyC1XJCpw96Jpjk0dYPT9/Yun4+mP2hyCgACU26oeb2rriglhDmgECGEJ54f+fSve++o4hEe5lnzmGrAwAPDC5g6s3daJ2RPLHe+rDuSG2kqsMW2XMoLDbWe0dlunY1CQskOOZz9tPdqDlzZ3QAcQIOCeRTWon1LhWGTUAaMmnOkwtBUZ5996tMeKUycC/tRyFJVlpa42dNVGKhcwtwWT4BRQdmFGAGZPKk8YoWWfmLqAa+lxv04/dTEO9YYd8QTNh7uxuK4KOzoHIp8a66rwh5aj5iJiHGvXrJa/3gwA1sIa1AglAUIkKqyoldajPdjU1oVb5012mFjcQgbP9EUQoIGYeCLEONm94vZbj/ZYfgM3CMBFE8vxf333Ujz5/l7rvBf6dTz14T4881cLMUEpwTKhfJgvM+KdC6rx4ucd7oIzy9VLpYanIgSgaYTlt9Vb/qFVH7dZNYsA92jAyrJSEBFICGgaEDV/Wgcwb0qF64ZllSIIAGOhrywrdYy/oAbY900PXTMD5SNKUFlWauWCSNRNYr5zDVgYmPRHdEwYPRzAwEKhOpABp33fS3qru6gbL5kQY790S+5pag/hH1/badndo8LYud1pZo7KY9wGjF2jeL/1uNmQgTj/qC5AACIC2NHZbUXZzJ5UHvN76k6xxVZ6wE5tVRkevnZmTB/YbbmqgF2zpcOxYMuFyE3zAIDqMcPx1zfMAuDuiPOKBnrhwUaHZgAAL33egZvmTLQEBAGYNbEcV86osvwhn+w9iZNnB8JKdSEcJohIVODS6grMm1phCQK7sHvu0/344dUzrH5VQwYJzgQ4XQCf7+9yxK17LQCqiUlFAHilqRNLF1RbiVhy8X5n1zGs3tyB+ikV1vslARoIwUxgRnyn5Wjc4IxEjtx0nc+a6SNRieoCod4wVm/uwKqP26w2qkXs7Av2ivXN1m8Jgbghxas3d+B/vLvbNWJLF8L4LSEQNX1Q37hkIN+BAJzpi6B8RAlaDnc7wmMle4714C9/sxm3zpuMUG84r7kGLAxMAhrh0etm4obZE+ImnUlfQFQXVh2U2ZPKHc7O+ikVju9cbwtfVW2fm9u60HK4G3ea4ZHqoNt/4qzDcXfXwmmembKv2KpJAsaAsjs41aG4ZksHvlk/KWYAqppL/eTRDtOGpONUL1a+6YxaUQWJKmB3HTmDnYe6rWip5sPdiOremSGHTl/A8td3QoBcVWs1Gsh+HYdPn3f8li6Ad788hpLggF24sqwUK95o8RRGRMCIkoAlsHQYwvTLI2dw54LqmAV67/Gz+OlrO/HNuRMd7ZpQPgz/+40XG9FVmzsc33l9+2EAxhisn1LhuQCo0W5uRG1BA6OHlziSKp/7pA0dNrOYbJvdjBiJCjyxYXdMVMsfWo56nrPU1JDcnLKpJmKpiDiSqOd8P/71nd0xwkqaRO3n14gckW+6cJpy7FqRXau1Y0SMGZqEfacf0Q2zo9VmAGu2HIQwQ4ADpkZpP/c6895/vOckrp01ztLg85FrwMLARIexoz3XF8HBU73o6Io1VzS1h7D89WZrMEV0gZ++ttOxA3u5qRPXX+zMipUVTZvaQ1YkiyRsOixf3noQP7x6Rsw5x44stUwn4ajA7mM9rpmydsegRJjnlk4xIZwJeTKePmBG02hm/R7V3NPTF3GYW6ZXlVmLirqD6TlvLD7ShGQXsCNKAthgRsSE+3X8/PVm192eeg3GZZnOS1PjUMs+uDlwn/pwn+vvfa+h2rILu/WbHV0HNnx5LGbnKx/s4rVAHztzAUFtIDb+6Jk+rHijBStur49JjJQ9kKgkszSFvLT1IE4rmfOSQEBDz/l+LHt6Y0y4b/f5fufiFRX4r7/fjlvqJ8Fcg6AD+HTvSWw5cMqxaN9SPwlPfeSsCEwY0A4BOJyyUrCUBDXc1TCQiNVnmqzmTxvjW0tYu60zbiLpxrYu13EkACtqb8AUGXucrgt8rboCy66ocbRnzZaOmGMDGuHqmVX4yEMoH1N8MvYM+GVXGpvLz/efwt7jZ2O++5EZlBEwTV/sM8gTMopEIge+/XkHXvH09rfCET0m01X6F3719pfwWnfCUYENXx13vEcAxiiOsNA5pxorBY3qGLTapgPLrjAWv57z/Y4Jvf3gabzTctRSX/ujAq1He2IiTzRyahUTyofh0OnzjgSrpvaQIxYeAK6cPtZyjEsn7Ed7jCxRAAkFgewD1Wn3sunEl6GRy2+rx5otHRgW1DBrYrllalHt4wCsLFF5jW4Cw46AdwAWYcAm/dwnbQ5/x7IravBB63FHiQQZ9WL/uXGjShHq7bf6IqILrHyjxTXXYfXmDjz7yX73nA6T+dUVeObjNtdaTF3nYrOqD3T1xizybkL+sW/PwdEzF6ydLGD0TXuXoR06Mm9tuQ/hiI49x3oc2sg7u45hw65jvkNU1QxrlYmjh0PTzriOJ2mSdCt/bb+OHZ3daD3m1HInKlrt1MoRuP7i8Wg+1O3yKwYXjR+Jpt6w02cIOHxBL30eK2TsbRFCZL3Aoxtazs84iFi3/ZDjtd+szsV1VQjYevaD3SewenMHtpilnb04H3Y+oU0AlhMUMGy8deNHOY6RnzV3nnb9zWCAMM80W8kdviQSFTHqvzR72HdTqgD7/EBowP4pDAFy37ObYpybH+05idWbO9DUHsKT7+8FANy/eDqmjS1DdWVZTFsvGj8S40c5+3hK5QgEA0arA2T4W+RiE47oePrDfVixvhk7Orvx+YEQVm/uwDstR9HUHooRkN+cOzHGRJHI1g0Y2kbA5cDyYQN7qZHDgtDMY4IBwuxJ5TjtMqHrJ492CIOus2GQYsDb0dmNe57ZhKZ2Y7w0tYdw11Of4aev7YwrCABDwKuCgGC0P5moYreInHPhaMxxAsZu/3hPH0qDGgJmhI+dUy5CSGBAu5LjQ16vnab2EN79Mr7TvKw04CoIAoqme/WscbE/YENqnbItj1w3c2DsacCJnj68+HlHjDCQlxsMEO64vDqm/6+YXmmNu1e3dSZMjNXFgIadS1gzADB1zHAcOh1bt2baWOeC5Vdaf7j7hGM7G4nG7ghVAppRfVJth12A6EKgzlZLCTC+s3pzB46ccd89XT97gpUha4ZUW4tCSVCLUf+l09zuN9CFe1a1gLG7lw5WN577pA1tJ84ZJhx4140KaIR5UyscO08AOBQyyjwAMBqvcOzMBYfpQ8DQ6n7z6X6HfVYDcNm0MSmp3pdONRzGqq1/1cdGv6k7a6Ebi9znLsK/7aTT/DhgBnMiI00A4PtPf2ZFvCQirKxEV06vxHWzJ8RohYmI6CImi3n/SfdwVgFjzK+4fSBjfvn6ZkTMiLa68aNco8QA4M/tIat8ikbAL82KqxK3cE313O8pGrVk3pTRjvDQRD4XXRhaZ1Qf8E2t/M48y8T5zq5jrvPgkknlON8fxfxpY/BP/7Er5vNT58JWO/b4qJIsx3BN1cikH+mZDiwMAIwbNcxVGHz3cmeIn19prT52UyPydMJKojpwoT9256Ues0HZJX3Qehzn43zvUKjXcjjbJ1WAgBW3GyF5NVUjY5zmrUd7MHtiOSaOHo66cSM9FxIZ4725rStmIQKcoaLx1rO6qrIYQSCx213VBdZrZ69GbmiBgWc8ALAiS979ynvXKZk3tSImKAAwFo/VLip/SVBzNW0IeBfLc0Nm2foVBCoaGdFSMldENbklQq2zNbasxPPY/oiOUG8YP7rhIjS1h6DBuDcajA2J9BWp2OeKLoB/VJ5F0FhXFROuqSI8VJ4Z40Ziyf/8xBrDqqavQjC0ZWFez9ptnZapNF7fyWs40NXr+vm+k+fQ1B5C69Ee1w2CF2u2dLAwyDX2uHM7aiXOjeZOLVkunjgKPX2RhMd5OaUkAY1iBEbbyXO46ZIJnoJm15Ee1wVTF4a/oak9hFBv2BE94oyi6Mb86tiFULZHxng//eE+tJ9ynwx+UHfMfmn12GkFNILQhSWABjKLOwAySocQIaHKDhjmgZ+5RJUAsQJ8ZGkA//7AIvzq7S9dj58xbqTneFNpOdzt6QvyQ0AjrDazqa+aWZWUIJDYv6P6r9Tjes7348n39xplHWx+qFBvGDcpJaa90AViSnHc9rUpnhsFALhxzkTXzwfe89ffGsEYG7oRTEGAI9EtVYR5TZuSXD+MSLzckRFhQES3APg3AAEAzwohHlc+/1sADwKIADgB4IdCiPZMnDubqJNHDZP0y64jPZ6LViImlQ/DUXNBIAiUDy8BMLC7HFtW4siCdMPLxPP7rQetEFd7yJ8aLrndY/HSdWEJlHQEAZD44TOe3/NYzRfUjEHLoW70KpPZHpnk14YebxEjcu4Z5V8HXKLRAMR1Pqqc6Onz5dPwQmpHEV0k3Gi4QYCjzlY82z0AK0PX3mYpJK6fPcGXMCAgJrwzniAAgLJhmdnTRm3RAv1RgZYk7lUies73+woNlgQ14NHrZmbs/H5I24FMRAEATwK4FcBcAPcQ0VzlsD8DWCiE+BqAVwD83+meNxfMU0wD6dycqA6MCCbf3UdtO8OIbiSx2BlTVorGuqqUFo1IVFg+gT7zQeuAe7KdGwKGKivLeSRDUDMiX9wcs8ngJQya2kMxgiArKBKlNxzFfc9uwtk+d9NddxKOwdO9YUfceq6xX5maVR/vePWwDV8d9+1vmzl+pMOvkyjRDgC2d/g3vSTDLo9ky1RY9XGba7i6F/Pi5Jtki0xEE10JYK8Qok0IEQbwEoAl9gOEEO8LIeSo3gQg+8/JywBWJq/J7zYeSOv3LqRq/I3DOJfwyWQwq1xb2atN7SHMnlQe9zt2ojrwro8dn0pENzSOVDUC6/we30+2lHmq9Lk0oK9fR4mHlCtNYkPQF9HRnsQCkg3kBiEdmX0+HPHlOAWAm+ZMdLyuGpk4gu9g6HzCY1JhVIY0DsAYj08n4cDf3tmNx99yNzVmi0wIg6kA7E826TTf8+IBAG9n4LxZR91xvN3snYXphxQKhsZwXtEM5k2pwNMf7kvJHiy/Lye6zF5Ndqd/Kg8x0YWMgDQfOdEIGD08vknPzuK6KtffySXyGeFuDnS/RKPCM+JHRTV5uuVGqGhZCpDvi+e1ToFk52gih3emyWmeARH9BYCFAP7Z4/OHiWgrEW09ceJELpvmimqS6c/Czj5ZQkrm6Qetx7EhiZ25fWkJBgiLzRo2MvOxsS55R6NbctdQp+dCrDlocsVwHD3jfxfbdvIcxo3Kb9/KBXHdnxNvECaNHoavzxoXoxWd6u3HmQuJAyiA2Ig9P5oBZUkLjBelV4xkQhgcAjDN9rrafM8BEd0E4GcAviOEcA2REEKsEkIsFEIsHD9+vNshOUWth5KJnX26qE34bN/JpJ75bP++EAKrzGxVI97d+NSraqYXwQDnLqq47RsOnb7g6Utwo+Vwt6e5KVfIR4V2+PBdyMWzRMk6i1d7SkWN2Nt64FTC7/T4FDTJElSz53LMMY/coWyRiVm8BcAsIppBRKUA7gaw3n4AEV0O4GkYgsCfvlgAFMDan5BzSSwuKlHdaVvXBfD0h/vQUFuZ1MCIFIDG5Jd8iy2vmHg3CN6x67lC+o8m+Qhz7D4fwcd7TsY47pPx36i+hW4XDUslXb+TF5RnYSCAnPoN0p4bQogIgB8D+COALwH8XgjRQkQrieg75mH/DGAUgJeJaDsRrff4uYJCnbd5HhuelJVkbomT8f6UxE8e7vafSJVv8i22kjFDTx4zImH5iWwj/Ud+NAMvklEce/t1xwI4tix/ZjK3Zyjkmlz6DTLiLhdCvAXgLeW95ba/b8rEeXKNnueJ6AcBo1IlMjRwx5aV4PG3vkwq67UQzGfFiN9aWNnkEzMuPpDGTihZP+y67YesApG94eyYgAYLNWNja3hli3xrzQWNOobzbUP0oi+Djq5ZE8vj1q53I1vRHEOd3SkmKmaSw92GwzuXY9++ALoVxxtK3HF57qLweRongVtMeSHgVhMoVeqnVGD+tDFJfWcQuQwGFfn2FwADPo7jZ3PnzLQvgIPJH5UNXvMRxZUpWBgwDpoPd2NkBpNtmMGNNAEmERCUNvaaYJnc6AxGvvBZxyoT8KxnHOw91oOOAtiRMoWB9diKHJ5zm8tzDYYqmU58iwcLA8bB/pPnYhLbGCaXpFrBlkkPNhMxDs5ciGCcj6xPhskWhZDpPxRhYcA4CEd0VLIwYPJIYcbsFT8sDBgHAsBXR/Mf0sgMYVga5AUWBkwMQzt+g8k3pZy4khe41xmGKSjCuYxjZSxYGDAMU1Cw/zg/sDBgGKagKNCqL0UPCwOGYQqKQVAfsihhYcAwDMOwMEjEk+/vRROnxzMMU+RwOYoE/PMfWxHQCL9YMi/fTWEYhskaLAx8ENUFfvraznw3g2GGDE3tIWxSnofMZBcWBgzDFBxL/7/P8t2EIQf7DBiGicv0x/4j301gckBGhAER3UJErUS0l4gec/l8GBGtMT/fTETTM3FehmEYJjOkLQyIKADgSQC3ApgL4B4imqsc9gCAkBDiIgD/CuBX6Z6XYRiGyRyZ0AyuBLBXCNEmhAgDeAnAEuWYJQB+a/79CoAbiYjzDBmGYQqETAiDqQAO2l53mu+5HiOEiADoBlCVgXMzDMMwGaCgHMhE9DARbSWirSdOnMh3cxiGYYYMmRAGhwBMs72uNt9zPYaIggAqAMQEEQshVgkhFgohFo4fPz4DTWMYhmH8kAlhsAXALCKaQUSlAO4GsF45Zj2AH5h/fw/Ae0IILkfFMAxTIKSddCaEiBDRjwH8EUAAwHNCiBYiWglgqxBiPYDfAPgdEe0FcAqGwGAYhmEKhIxkIAsh3gLwlvLectvfFwDclYlzMQzDMJmnoBzIDMMwTH5gYcAwTFxGlQZyfs5Hr63L+TmHOiwMGIaJS/PKW3J+zse+PQd//63ZOT/vUIaFgU/mV1fkuwkMM6RorOO81FzCwiABY8pKcMf8KVj342vy3RSGGRLIRamhtjKv7Rhq8PMMErB9+Tfz3QSGGVIIrlqWF1gzYBimoOB01PzAwoBhGIZhYcAwDMOwMGAYpsAI8qqUF7jbmRjK8pBkxDCS0SNK8t2EIQkLA8ZBUAOGl/CwYPLHvCmc05MPeNYzDiI6T0Ymv3AwUX5gYcDEcLj7Qr6bwAxh6iePzncThiQsDJgYzocj+W4CYzIU86/K2WeQF1gYMA40AFPHjMh3MxiTfJtMpPsol0KJaxLlBxYGjIOSEg2zJpbnuxlMgRDVjf+1oaiiDDFYGDAOgkS4c0F1vpvBFAhSM8lliYhNbV3W3yyEcgcLgzgMCzhH4lAYl8NKOMegUCiE8SZlAOWwMXYzkZ5vO9kQIi1hQERjiegdItpj/h9Tc5aI5hPRRiJqIaIviGhZOufMJYGAs3uGwrisqRyBV7d15rsZjEm+BUKpuSGqGzcyZ+d8p+Wo9Xe+r38oka5m8BiAd4UQswC8a75W6QXwV0KIegC3AHiCiMaked6cUFnmjGoIFODIzHSbjp65MCSE3mBAABie59oMcpd+vj+a8m8kO0b/YBMGI4exppor0h1pSwD81vz7twDuUA8QQuwWQuwx/z4M4DiA8WmeNyeUKSaTEQVoQolmeOWuLCvlpLMCYURQw9jyYXltw+HT5wEA3ef7U/6N4UnOm1vqJ6X8XSZ10hUGE4UQR8y/jwKYGO9gIroSQCmAfWmeNyccNCeCRM9TOxIxeXRqC0bA5e4vqK1E8+HuNFvEuBHUKCmzR78uMLVieNba4wcpBKaNLUv5N0YkUetq/KhSPPbtOdbrfGtG+aY8h5pRwp4mog1E1Ozyb4n9OCGEQByzOhFNBvA7AP9JCOG6rhLRw0S0lYi2njhxIslLyTzTlHj7QhyWAQJ6Iymq8C53684F1WynTQIvE4jb21fNrMLEJAR3iUZ5N9nJ81dXpi4MLq+pxKzx/nwOJ86G0dQesl5PGeI5Lz19qZvnkiXh+iaEuEkIMc/l3+sAjpmLvFzsj7v9BhGNBvAfAH4mhNgU51yrhBALhRALx4/PvyVpkjIQSwuwgNuo4UGcu5DagFFNTGNGBNFQW5l0aGkh+lJyhVfIpdvbH+05icqyUt+/PbxUwxFFO801cmeeqlAKEPDodTPR0+c/q90eWppvYTiUSHd1Ww/gB+bfPwDwunoAEZUCeA3AvwshXknzfDlFrZHy/YZpeWqJNwSCyNCUOXMhgqb2EBpqKzFnkv/Es/Gj8mvXzifJ9nyoN+z72L5+HWeTWESzQflwI4iiO4l2A0YoKsGIyGs92oNjZ/p8fU8jZ2hpqDd1X0WhUXhbSSfptu9xADcT0R4AN5mvQUQLiehZ85jvA7gWwP1EtN38Nz/N8+aEM3meiH7QCBgezIxdURfGrqypPYSvjvb4/t7Jc2EMG6K23dFlQdf3vbSlZEotaETQshDgP2dSOS6r9hckUGre11PnkhMGQhiCsj+iY82WDt9C88Y5E9FQOxChPiOHIa3ZJpUtWy617rRmsBCiSwhxoxBilmlOOmW+v1UI8aD59/8SQpQIIebb/m3PQNuzzp9ttksA+P3WgzlvQ6IMzO8vnIb6KZmp8kgwFqtNbV0xAzdeM3QhsGjG2Iy0YbDxD9+ag3/67qUx98ltDb9j/hQrOscPOoCaNBy3XiyorcTy2+t9HSsX4xnjR6V0LgEgHPEfenHD7AlxXw9mSlJY2UvcojyyxNDczvlEVekDeciNjxdnTQBurp+EBTUxuX4pUVVeiobayhi7NiF+BmrVyNJBFQJov42pbrwvmjAK//TdS3HvohrMnlQeIyx1Zf27dtY4PHH35eg41ev7HCNLAzh6JvPlxOunVPhOLNxibogevW5myufrj/oXBmok2wetrm5IB7malhoBUytTd2iPTcJfJLmQhCBNFxYGcVBV+myqrKVBzdVOP3Occ0dmP0YAWLutExttDrd0OGVGcoR6w44JtnB6JUqDGgJkDJhSZYfzf9w8G5v3u7eh0HzLk8qHWUJdI+CK2tQE6cjSAO5dVIOm9hDueWZTwnyPfSfPAUjOTHR5TSXGjkx+AYmHBmOTc6LHnw1fljNvqK1M2WRRl4RWoZ7imA9hmK2SFRoN7OYDGuGXd1yK6y9OPbDlmM8+tzNozETFjlq986IsVfO8aMIovPhQI65zGWj7Tpx1vC4f7rRRE5Cyvd5tnG1q60JjXRWCNmnw54Onsfy2evztN2fjl9+9FH9z08W4Y/4UTK8qw6PX1uHeRTU4c97dvzIpxRyIdPHS4o6f7UO/uXLrAhhTVprSA9jbu3rR1B7Cq9s6Xc0gQjm9XFSTqQh7/ewJOJfh0MLSEg2NdVW+7dfXzhoYk5VJCKbasWX4+qxx+KfvXopHrpvp2cd24RzQYosk5tNnUDduJB64egamV5XhoWtmYPakcuw+5t+XZkej1HwGuazNxMIgDuoubmmWqnmONJNynvqoLeYzUuwYfREdpUENBEObuHNBdcolpwMBciSeBQPGQtFQW4nrbbbaSFSg+XA3GuuqsPLNFvzLH1uxbvthHOjqxXOfHcDjb33pOtDvmD8Fl1aPSalt6VAa1HD71ybDLRJYnVwCwMollyad3HP6fD/uXrXRc4ddoggjmdXtN7SUYOzgj3RnLrS0esxwLL+tHg21lZjgM7PZPraSqU90zaxx+N0Di3Dvoho01FZizSNX4ea5E3HR+JEOrfNoTx+i5k2J6gKtSuBCV5KO60xy9kIET33UhgNdvXjqozbcvWojthwIJfxeQHOarr45dyIWpqiB5tIyzcLAg1yaNxbXVWGthw333itrHK+XXVGDFx9qxN99azZefKgx6bwAsv1fP3m0FSdPAL7XUG1FcoxXFguCoTWEI7pj4Q9HdKzbfsj1XPl4LsKV0yvxw6umY932w+j3YW7t7g1j+evNKSX39Ee9g3qDiuNv15EzAPyHlgY0QmNdFSoy+NSvztMXsOKNFqze7C+6Rw3zTIQMJ5WbFDsNtZV49LqZ+O6C6rhmo+c+cW6Ibp03OeF5Rw93j+hKl7PKE//6fdZ+uXiC04f0we7UE2iHJ5G9nS4sDDwQcCa/wOV1pigfUeIqfOZXV+Cxb8/Bo9fWOUwyDbWV+NENF1kLd0NtJeZOTrzwzq+uwLASw/Y/rETDsitqLF/AsBLNofncuaA6RgNprKuyQg3teO129xzr8eUAzCT/cOsctJgLrxclAaMsRDBAaOo4jUiKunhJgDChfJjr7k219Z48G8bqzR2xznkynMsqD14zAw21lb61UVUT8SIc0fHz15vx4uaOmM+mVo5wXItqajuVIOZ/yWVTcM2scVhxe70jPBQAmtpDuO/ZTfj1n1px4ORZj18ALigF8e5dVIOLJsT3OSSTyAcAZT6TR8cogjjgs5zIl0d7HD6kcETHoRSTByOZLj4Wh6ITBpm8IHWQNdZVZUVjaKyrMspAKD9ePqIETe0hPL/xADpO9eL5jQccqfp2EpULuGjCKPz89nosv60eV100Dstvq8e9i2ocr+0TuKG2Eitur3dM7obaSiy/rT7GD7CgthJXTo9Vg9dtP+y5m/Lbj6OHB11/24u12zoTPlD9+tkTcM2scfjG7AnQUxQE1WOG46WHF1tC0w9vNx+JiZYRAvj8wCl8c+5AWS+NBp4D7Pd5wBOSqGEU1d01mnuvrME9V9ZY90bXhWMD1J8gsuXNL47g070nsfLNlphxKrVKXRiRVl495lZ+Yl6C0OmoGrqVgD6fC+zZ8IBgIgDLrpiGm+bGLb/myZHu1KLCcmmhyI5+lUcyGYilqvQNtZWYOHoYjvrMpvTDnEnl1iK85LIpWLf9sPVZ1chSPLFhtzWJ+iM6NrV1ue663kuwA993/CzuWbURIEIkqmPLgVMAgJVvtqCvX8fGfcakv3dRjfWbK99sQTgycGzz4W680tRpLQoEoMRmErjrqc9cbfJ2Ahpw9xXGOV5w2Z2qNNZVoaw0ACCxrRYAXmnqjBvxEQwQPmg9jkhUIBAglAQ1RCI6NI3w4DUz8MYXh3HodOzE1WggkaokQPi3exZY9+GFBxvx1If78M6uY9bxcyaPxueKfbl+8mhs64i9jr5+w/QW1AhRXUAjsjYifs00UyqG41Ao8e4zQEZWcCSix8yVyrJSNNZV4WXzHgdMH5IkkYlLCpmwyziVWmV/RAcRQTc7U4PRp3KcbO/strLgAWD15g7HnHDDT/ilXFQ1guWjSMTXplbg8wOnjL4wtQK/vpYYUtzgB3PoNCg6YZAp3OylTe2hjAoCAJhtCxVVbexvfnHEmmCEATuyyqvbOhOqkwJAOCqs4hX9ER1vNx+xFqKILrD89WbMNoWTfScX7tex/PVmx45SA3D1rHH4yU0Xo6G2Eqs3d7hGPmgU67S9c0F1jKPQiz/ZFlg3aseWoeNUr9Wu/ojuGY44flQpLq+ptH4zEhX45twJGFc+zFos7IIgoAENNZXoi+ioGFGCj/ecBGDsmFuP9jjMdM/81UKs3tyBt5uPWHZuVRiUjyhBn1vkEYB3vzxm9VNEF1ixfuBezJ1cjl1HvPtLI+DiieVoag8ljD4JBDSsuL0eod4wdhw8bfWFDDltPdqDqMwLUAovxdOA5ldXYHunofXoIlarllrl281HUD95NJ77dD/6owLBAOH62RPwzq5jEACiUacgebv5iHqqWBKsl0HNCBII9YbRc77fNVDDjUV1Vfibmy7GUx/uw3tfHceLn3cgqBlBF16pE6OHBzF/2hh8ZI4Vqw1BDfOrK2LGRCL6ksjRSJeiMxNlCs1FIqfjMyACHr22DmOUB+as234YP3ttJ5raQzETyL74CvkjLiSz6SgxfQQlQQ23zpvssAvrYsAsIHdyATL6QhdO00IwqFmCAACe+3S/e9uUxkV14OkP9yHUG86ICjxx9DBHuwQMh3zQJUDbLTJFwBCmL37egaeVRWLS6OHY1nEaOzq78dGek9Z5ogJY/npzjCnk3kU1VgTN+4qmJjcXy65wBgRI1EU8HBVYu60Tqzd3xBUE8rvlw4IoDWrWhCa4r5ERszzE4dPncf3sCRhu+pBKSzRUlpXi5+t2ImpqQBHFTCQ8qvIRgC86neYv9fqlpvnp3pN47tP9iJrjSYdhtpO+rJKgUxtxcyCPLRvwsZUECMM9snQJwNdnjcOaR67CvYtq8KMbLkpYMK8kQAgQMNwMwW092oN3vzyGqC6MjVFUoHqMt0n2G5dMwKf7YteJ7zVUpxSansvQ0qLTDIiSf3i3Rkb0R78tUkaPihhVN1lHlZ0ll01B+YgSXH/x+Bi194XNHXh560HctXAaCAOLe8A0G1iLUNTdTLR0QTVW+zC5zJlUjl9+91Irl0D+zvLXm6ELgVLbRGyorcQLDzZiU1sXKstKsWJ9M8I27UNXO9mj0wMaxThot7aHrAVAaiap4lYz50xfxKzp4/xlIYBx5cNQGiD0R4XlAJYakMqR7guek1EKTvu9aGoPWX27X8kPGTuy1PK7/K9NBxIu8ICxmPnaGQPY2NbluF+h3jAqy0rxj+t2WtdAMBbfHZ3d2NHZjdLggJbQaEa02RVMogFNtKk9hK5z7g5kt9r16vW/uq3Tutf2cSTDlmXb7eMSMARsR9c5PP1Rm3WOUG8/NDLqGD1y3Uw8/eE+dLqY9koC5NiwNLWH8HKckjIagLsWTsPUMSOs6zbmhvO49jhZ5K9vPxzTFxoZocWzJ5Xj91s6kExS8Ygc1vwqOmGQyspSN34Ufnj1DLzfetyy++qIXfz9hwUCv1hyKV77cycOnupFY10V/tByFOt3HPYsPBaOCuw+1uNo/m1fm4yyYUG80tSJaFSP2TVJGkwHbiIVdIG5GAEDWo4sp+A2EeXi1dQeiunWSFRg5RstWG46l394TR1++trOmHPe9rXJaD7Ujb0nzlnvnToXxso3W7D8tnpLdfey1Seibvwo7O86Z6ntMgIq4qJeBwKEpQuqsXRBtXW9rUd7oBFBiFinarxdWalyL2S0TNjMA5lR5YzJP3XOyO5uPdrjEAQEYI6LGUhGcLUe7bHMU/E4eS6MV7d1on5KhbW4v9Ny1HENMyeMwr7jZx0mtebD3ZhqOm3VkXnjJRNixotfxo4sxZPv77X66OWtBwdMjIrpkDAw1ty4uX4SnvvsgJXcJ2BoZ+99dRyPXDcTj1w309WcKAC0Hu2x7vWmtq644aGlZkSdbMeT7+/17V+wn1NFF4Zv7oUHG7Hmkavwj6/txJc+zaSTcvhwo6ITBqnsMvceP4sVb7TgroZqa6BqFLv4G5m5SCjZZaLYjoOn0R8VeGPHYeim6h2zo7YRjugOzeDNL45gzSOLHYuX14T5h1vnYNnTn7m2ze7oVRetFx5sjDsRAWNX5zaJdnR2475nN+GFBxstx/Mv3mzBeVuA//6T5/Cr710W07b+iI5Qbxg/uuEiNLWH8Mwn7mYmwHD6Sp9IMEAQwjChlZj25ve+Mu3eBKy4vR6zJ5VjrW0nKpFdbxdyK99sMZ228v54NsM4B4BLqyuw7IoaR5/ZfSz9Zt/aEcI4Rl1Uv1ZdgeW31+OeZzZZi93YkaX4vpnzIc+xZksHzoWjjsXczqHQecshr5HheOxXLmZkaQAlAbJ25oEAYc2WDkT1Abu6dPKWBDU8YqtH1FhXBQ3eARpXTK/Eto7TVl9u7+zG1vYQSoPGAiu1Q4Kxo/9w9wnrPPVTKizB4TYON7V1uQr3qGnGaqyrQoBin8/RHxX4uakZlQQIP7x6huf6cJl5H1Sn97ASDWFzPNtbcMf8Keg6F0b95NEOrQVw95PJ4I/GuqqksphvmpNa9FIqsM/AJBzRcaKnz7KTqzs/wFhEpo9zj3kuDQ7sqyJRgec+aUPYTEqSNthELK6rcrXhq3kFbtizPO0QjAVHhoeqi5afHV+8tof7dYeW8fVZzkieiaOHW227d9FAXkNJ0LBRP/n+Xjz94T7PHRjBqMx676Ia3LeoBiu/Mw8BGrCJf9B63BIyujAinqSJS32qWFSxgatJdDfOmRg3eqN8eBCBAGHnoe6Y8Em7j6UkqGGxMnaCAcPkotrApVBZcXu9NRlPnQvjqY/aLNPfvYtq8PqPr8Gvln4tJvbfrbW6MBPilC6dMW4kXnx4MW6eOxGXVVdg1vhRlkYV0Q07v5rQKGmorcTD19Y5fi+gmUlmAcJ3L6+G/blskejAGBOA5c8IaISZ40biuovH42vVFfjhVdOx8s0W/PpPrbjv2U2uodOyb9VrlQEVm9q6XGtDaTQw98JRgY1tXVYehQzIkHk0qiCQ1/zCg434r9+ajZf/81X4p+9eapXYeOLuy/G7BxahpmqkY34QgLuvrMG9i2rwzbkTHWMiXlu98BtanAmKTjMote18kmVc+TBP26WkbtxI7D0emzRTVhJAODLgnEr2AeIytnzlknnWbiboET3kRUNtJeZPG+MIcRQwnHutx1owe1K5I8TPy+yksnRBNV7ZehD9UYGABiyoGTBJqea062dPcKjssqyF3OVKLaeyrNQKXY0nbQIaxajuEdOPEtEF2k6ecxx/0iwP0VBbiZqxZY7oLzVCTO2LR6+biRtmT7CcqCpnL0QcJha7z8DuY5H2dzvXz57g2OnLqCOpUYV6wzG77rebj1ify3OsXDLP4QcAELMrtmsGdoEgHegf7zlhCEHlGo+fuRBXS5TJkQLGLnLZFTWWfX3ttk6HUA5ohAAESkzNYN6UCisizR7N03z4DIQQcUOnZSSS/b5oBKxcMs86VvUQVY8ZjrlTKhxj8VxfBMGAZplcpZkynsZt74+G2krH/QAMjc2OgOEfsIdoq+uJ3zUq2QzwdCk6YZCKGCAYaqRcdOLtwB+5biY22MIAJb1K7YNSpaRzwAxULwlquHL6WHxxqNsRx2xfmAMBDXpEt6KH3AaUFz0uQkhgYKL96IaLEgo8lYbaSrz48GKH7XWrGcaomtNkxVMvU5vs3yff3+vLefzgNTMAwDIjVJaVWn2vCyOyxM44Wxz4GMXns1C5t+oCDhgLsFeb5PvS7BZvoqq7WHt8+r2LamIWFTcTpKpFyIqyD3+9Ds9+st9y+i+/rR7Nh7uNEiOKz8C+8I4oCViF9XQR20avSCd7G60ADYJDSD/94T7HsTdeMgGXTRtjjbFNbV2uyW5RXSCoGc/ri9enod5wjHPbHpatBo5UjRqG62dPwAe7T1jmt7aT5xDUCHdfWYM7bW1PBTkn3YpErnyzxQoLVtcTOZfWbuvEGzsOo+eCd3RTsoEw6VJ0wiCapFYwqXwY/vKq6UktjL+841L87LWdMTV6JMEAOWLdNQIeumYGykeUOHbEnx84FbM7efL9vYhEdSvmeu22TmsC2238bjz+1pcx4ZF2IWSPFEp2Iqjf8dIu/GoelWWlCQUBwYgMsvs41PIMFcqCLwvCAU7BALhXnbX7D+57dpMloKwdsLnL1jFQGkDTKCZjW5aylte94vZ6Rz8kqh8lTWlPfbgPx89cwLIrnAJD9fWsXDIv4a4WAJ75ZL9lgtvw5TEENXLsju9fPB0tR844tBQvfrfxgEMQ/27jAavv7GVHSgKER66bGROJ53a/1Ygmr2txC7t+YsNu/OSmi42HMSk/vvOQoQ2vuN3Ibfh070nopq9pypgRKQkCKQDsczioUYyPwG5+jReY8ef2UFxHsjC/n47QSoaiEwbJpmgc7enzLQgkctL83FR77YwaFsBVM8dhw5cD6qkugOc3HrB2oVYyl82JKlEXUwJ8ZSCrGbCAsYj5XTSSQd1Rx9tte51T5hnEEwgChtnH3l+q8607jiZiFwxur+3Y/Qcyoe7WeZOtHfeeYz0D0VpCxGg89lLW4YiOlsPdePGh5DQwAJhv2027tU+OA3XcuPHqtk7H+JSL4bIrB8InkxkTasG1DV8ew5Pv78Xh0+cdDuK7Fk6L+V31fl9WXYF5Uyt879Ddxsune09ii7mhko5xqe3Y++knN12MLbZM4sOnz1u+CXsobrz+sAtjwoBZLhIVjvQfzeYPcwvUsGv5fh76k044e7IUnTBItMC48dSH+/DMXy1M6jtSIPz3t3eh58JADZOzfVG891WsGUku5KqZo+d8vyOSws10sXZbp+dOWw7SCy4lOr9xyQRLXc008bQLt89UU5dfzcAe4aOL2Aeky5Lebv2TyGRlRxXCP7npYgAD5TrsbVXLNACxY0549IMXXlFeXu3zY0tW2yTNW0sTLMBeZsnpY8uwvbfben0uHMWv/9Qao224FddT77eq+cSjqT2EQ6fPW6VDYJqE5ILffLjbNKkaC3NAI+j6gDYs59TabZ14pclIMHx560GAyHJwa2bQiJfmbRfGjj7VyErG0wBcfdE4S1txC9SQ9zioka/SFn7D2TNBWsKAiMYCWANgOoADAL4vhIgNBzCOHQ1gF4B1Qogfp3PeeGhxUsW9OJ7CowWb2kMxiVgS9fx2G/Omti5rkSPAYfu1h3naB2S8nbYcdG5s+PIYPtpzIq5pKZN4LSJN7SHcs2qjlej14sOLHTs9gjGX3QKKJpilIuSOfYbiwF92hXeeRDILqJtG8+T7e2NKdhOcpb4ldie79D8lg9viIc8h+9WPw9OOqgndPHdijPlGJZ5QUiNbZK0mP9pGi61AnwYj6iteOKlbe4Ia4Z5FNaifUoGVb7Y4tGdZL0sXAOkixi8g/RYyysmYt06tyUvzBgbGkn1jQDA2XB/vOeHYRFiOYmXsOawCUeGaKKcymDSDxwC8K4R4nIgeM1//g8exvwDwUZrnS8jI0iDOxHHKuKGGAfrBLYFFLmrSxhw1o2+WXeEcmMNKnAW7Eg3EeDtMOUjdNINEv5tJ7Db3gGmekju/tds6LaEpyywsXVBt9YOM7Gg+3I2XPh+ocSRgRIDYj5MRP2o0jle/JeMsV/tZ9m243yjqJnePbgu96mT309924ekluBJpDPGw7yoJwGXTxiT8bjyhdOu8yY4EuJIAIRoVICJHBI3bddozf7UA4ZWmTkSiia9pU1vXQP2sqGHvVxMlAWDNloOWqUoXcH3WtHyCn30DZ/cN2cOdvcyfUruQWtCj183Eo9fNjLnvXmNPFSiJGDSaAYAlAK43//4tgA/gIgyIqAHARAB/AJCcPSZJvnHJhIRVDlVSieVtrKtyJPAAclAZyS3PfXYAUQhomhazQ7EX7Hp+44GkVH8VOege+O0WnFZMKHKA5yI8zTFplaJ3atQKwXuyvPflMUc46PaDp2OOcwvx8yIVZ7n9u2qJB79hiIlwW+Td+iPe4pwI+65SwN8uM542JUtD/KHlKG6pn4SaqpFWKRN7BI3KprYuh0+hfvJo7DzU7eua7OYlexiz2tcrl8xzFFOU/gS7oGmorcRdC6dZD/ex+4Zk+Q7pGHYTUvKcSxdUY+22zoS1tdwiiewCxUujV68/V6QrDCYKIWTxlKMwFnwHRKQB+DWAvwBwU5rnS0jZsOQvacfB046yuX6QO8GVb7Rgh61Il64LtBw544gIUlV+e2noZFV/r7aoNY9mjR+JSWNG4NZ5k3NiImo0E+YGdmcDCXN3Lqi2yiLbI2vcFs875k91hEPeUj8p5rhkQm3TJR1hEg+3Rd4tsTAVX4EkGZ+JJJ42JZ+tEY7oeH7jASxdUO1Ls7VrWJpGWFxXhdZjPb6uye81zJ5Uju9fMQ0th7rjCpo7zYXczawjzYJ+hJQMGJC+Bz9ajuxfKVD+8jeb0BuOLxAKSjMgog0AJrl89DP7CyGEICI37eevAbwlhOhUn+frcq6HATwMADU1/nZ+KntTeGD1n3albluvn1phLv7GpctqoFtc8geA1KJC/DBrYrnDeb6/qxf7Tp7DlgOnsuJEVhfkhtpKa3fmVvTOK7JG/Z2b6ydZ4ZABjXBz/aSY86ZqNkn3GjMphNwWebffT9bUlegcfvASgOrYFfAOMVZ/b/lt9dbYeH7jAd+bID/XYPdJBTQ4nNluVQTU/rSHjPq5Hkc/mL4HgeQ0NxlE0Rv29hsQCizpTAjhuZsnomNENFkIcYSIJgNwe8LKYgBfJ6K/BjAKQCkRnRVCPOZyrlUAVgHAwoULU0q5cKti6YdkbmRTe8hS9SJRw7H1zbkTMb58mGUSkvVxVPGXzk4vHrKOivRFWA8a6c+8z8BrQfZT9C7R7xgx49JpEFsZNB2zSTKs3tzhEGzLb6uPa0JIFnVRAuAp5FLVTuwmyUxoiOrYVYv+xfv9UG/YoUX43QT5EYZ2n1REB26+ZLxniK78Tbumbu93P0LK3g8BjQCiuIUkvaifUhG3OGOOc87SNhOtB/ADAI+b/7+uHiCEuE/+TUT3A1joJggyxdiRpcCJc4kPtJGMbV1NTgKMaIrLpo2JGdxSlVy7rdMRKSTtholsjslgnzT2B3i4VV91u6Zkdp7xFuRkFi6330kkLBvrqqxy424hnpmgqT2E5a83WyavcMR4EFCmhZC9r5IxUSRzHSvWN6M/KrC5rSttDdFrYfbzm+lsghKNKXUeTSgf5lvbTkVTdxPkqWhuaukWr/blwswLpC8MHgfweyJ6AEA7gO8DABEtBPCoEOLBNH8/adQSBImQj2H0m/yiFjczfiO2hlCiHayboEgXOWmefH+vb1txKmaXTGk3br+jOm1lfLajTTbNIRvI0gkSjSiu6S8TZENjVKO47CXHUyUdLSVVc1civHxSfsiUKS2V60nkDxhUtYmEEF0AbnR5fyuAGEEghHgewPPpnDMRagmCRAiBpNLT1XBDAK5PIIs3yLJp6mhqD2H7wdMgImhw2u7dSKUtmZrYiXaabkJKRqbI+PZs7JykyU06PGWYrJcJLBMk26d+tDl1VNpLjudqt2knU854N39VKtnesk3ZElKJaKyrci13LfnOZVNy2p6iy0BeuqAaaz7v8F0mVqPkK4O+8GAjntiwG5+Yj0KMuDyBLN4giyco0nFSSkeaVa/epYaOSqadjMni11kp+zdbPhe1TV5CKlOT0/68ZHuuhF+flR9tTu6Y7SGMuco7yRbxnseRzqYk3nezGb1mz3uoLAvidO9AZdy3mo/iL5OMckyrLTk5Sw5pqK3EQ1+v8/3Q6wevmZF0ZzfUVjoScHThbpf3GmRei026kTJqIpyux9bQ8duWdHBb6BJhn3AAcOj0edeokGzsoN3I5MKvsnpzh/VUODmG/PYT4F+bkztmNVEql6aHTJOrAAJJNqPX7PkXAQLmTR2DT2xJfbkW3EUnDADgzS/8J50lekC2F8229HrAqGuejIPObbFJd6CriXB+J36md7zJLnRqyQEZt+1VbjjTO+hcoz7XWH1uQSKSLbPRUFvpO+qn0MmFZmgnm8JHvZZb503G5raupOdvpig6YdDUHvJV80OSqgtSPkRFkgl7bLoDvaF2oFY6AWnXbE+FVBY6r7jtSFS4lhXwi596P/lYHNWyDupzCxKRijaXTU0nl2RKk/V7/7MpfNyuxR6Snuv5W3TCQEaf+CGgIemCYhI3R3W6Mf2ZGOj5nvSpLHRucduRiOGgdysr4BeviZxvjUEKx2RNaRK7Ga4YFvhkSXeMx6uj5XaubDqY7dciBVSiqrLZouiEQWOd+8Ox3dASZETHw602vp+Y/kTkezFPl1QWOnXCAcATG3ZbDyRJVT33msh+Vf9sag9uTzvzQ7r+hmIlmXsVr46WG9mckzKB9WRPHz7YfcJ3WYtsUHTCoKG2EjfOmZgwmQMwzBCp7uTfb41NtibktpaISj5NH3ZSWejUCWd/IEk66rnbRPZb4qAY/Q3FiNe98poP8epo5brdy57+zPGoUyB/EV9FJwzUR/DFQ9Mo5Z282zMQ/FaGzAaFunilw50LqrNiO/Wj+uc6asUv6fobihG3ewXEL+/hVUcrl6zd1hkjCAi5dxxLik4YqOGVBGDO5HLsOuIsYGc8QDt+6d14LLuiBjs6dzre05A/zcA+IcL9uvV82EJYwJJFFWzJZJT6JZHqn+uoFb+k62/IJ9nSXN3uVSJhnu0kQj+oRmqNgHtcoudyRdEJA5nVJ+WBAGIEgfxAR+q7Plnbfd32QzhxNhzz0Plco2ZGp+N4zSdN7SE8sWF33nfl2XYcpkOq/oZMkOqCnk3N1eteFaIwt2NPCgxohF/EcWTngqITBn59BvanG6UyUOy13TUySlkvu6Imb4uGnBCZcLzmA3slWPtzafM5kQe7Mz/TpLOgZ9vspt6rRMK8EMyqMimwUDYcRScMAOCR62bi3a+OO4qNqRAGHl6dyk2wD25dAF90dqP1WGomp0zRUFuZMcdrLnGrBGt/uHi+JwljkM6Cng+zWzxhXig+oULacBSlMGiorcQvlMfgqQQ0SitOu7Ks1FFgSn24Rb4iewrZvOGFWgmWAJSWaCwICox0y1AX0rj0ey2FEqGXC0hkqQxwuixcuFBs3bo1rd+Q9md79AVgOI8JSEs9fPL9vfiXP7Y6FrBhJcbvAd6RDEwsUjOQSWd3LZyWMSfaUJrMuaCY+jPRteTKlJTpPiWiJiFE0s+aL0rNABjo4FvnTcbGfSedIVxpOo+BgTpA8lF7y2zPRMjGg0qKmWztGgvBLlxsFJJZI10SXUsuTEmFNEaLUhioHfyNS5wOZU2jzET/EAEQ0DTNsZMt1LDEQiYbi0yh2IWZwUku5nEhjdGiFAZqB48rH4ZSWzVPIudOPtVzRKKGnTuqPM+g0OyjQxUWykw65GIeF9IYLUphoHbwUjOT9YXNHQAAPSqSerqZn3OoN7GY1OnBCgtlJl2yPY8LaYwWpTAAYksZtB4dSDzTAew4eBpNaTxFqJBuIuMNC2Wm0CmUMZqWMCCisQDWAJgO4ACA7wshQi7H1QB4FsA0GFGY3xZCHEjn3F54lTII9YYdzxt9Z9cxfLTnRFoOm0K5iZJiivRgGCa3aGl+/zEA7wohZgF413ztxr8D+GchxBwAVwLwV0kuBbyKVkmzjqwHYs8LKAakEPz1n1px37Ob0NQeI5MZhmE8SVcYLAHwW/Pv3wK4Qz2AiOYCCAoh3gEAIcRZIUTqj69KgFz0A0opA2nWuWdRjevngx0vIcgwDOOHdH0GE4UQssD6UQATXY65GMBpInoVwAwAGwA8JoSIpnluV+y2/MqyUmtRlCadhtrieR6snUKKSmAYZvCRUBgQ0QYAk1w++pn9hRBCEJFbOnMQwNcBXA6gA4aP4X4Av3E518MAHgaAmprUq/fJBT5ePfNiEQKSYnJos++DYXJPQmEghLjJ6zMiOkZEk4UQR4hoMtx9AZ0Atgsh2szvrAPQCBdhIIRYBWAVYJSj8HUFHhRTfX+/FIOQK6SMTIYZSqTrM1gP4Afm3z8A8LrLMVsAjCGi8ebrbwDYleZ5EyLNJhpg1fdnx2rhw74PhskP6QqDxwHcTER7ANxkvgYRLSSiZwHA9A38HYB3iWgnjJpuz6R53oRIs8nVs8ZZIaW8uBQ+XgEADMNkl6KuWgoYZod7Vm1Ef1SgJEB48eHFbHYocNhnwDCpw1VL42EWlDP+ZwqdYvB9MMxgI10zUcHjVlCOYRiGcVL0woBt0AzDMIkpejNRMcXfMwzDZIuiFwYA26AZhmESUfRmImZw0dQewpPv7+V8EIbJMUNCM2AGB5x9zDD5Y8hoBrzjLHw4+5hh8seQ0Ax4xzk44MqrDJM/hoQwcNtxsjAoPDjyi2Hyx5AQBrzjHDxw5BfD5IchIQx4x8kwDBOfISEMAN5xMgzDxGPIRBMxDMMw3rAwYBiGYVgYMAzDMCwMGIZhGLAwYBiGYcDCgGEYhkEBPwOZiE4AaE/jJ8YBOJmh5uQKbnNu4DbnBm5zblDbXCuEGJ/sjxSsMEgXItqaykOh8wm3OTdwm3MDtzk3ZKrNbCZiGIZhWBgwDMMwxS0MVuW7ASnAbc4N3ObcwG3ODRlpc9H6DBiGYRj/FLNmwDAMw/iEhQHDMAwz+IQBEd1CRK1EtJeIHnP5fBgRrTE/30xE022f/Tfz/VYi+lYBtflviWgXEX1BRO8SUa3tsygRbTf/rS+gNt9PRCdsbXvQ9tkPiGiP+e8HBdTmf7W1dzcRnbZ9lq9+fo6IjhNRs8fnRET/w7ymL4hoge2zfPVzojbfZ7Z1JxF9RkSX2T47YL6/nYi2FlCbryeibtsYWG77LO64ymOb/97W3mZzDI81P0u+n4UQg+YfgACAfQDqAJQC2AFgrnLMXwN4yvz7bgBrzL/nmscPAzDD/J1AgbT5BgBl5t//WbbZfH22QPv5fgD/0+W7YwG0mf9Xmn9XFkKbleP/C4Dn8tnP5nmvBbAAQLPH598G8DYAAtAIYHM++9lnm6+SbQFwq2yz+foAgHEF2M/XA3gz3XGVyzYrx94O4L10+nmwaQZXAtgrhGgTQoQBvARgiXLMEgC/Nf9+BcCNRETm+y8JIfqEEPsB7DV/L+9tFkK8L4ToNV9uAlCdg3bFw08/e/EtAO8IIU4JIUIA3gFwS5baaSfZNt8D4MUctCsuQoiPAJyKc8gSAP8uDDYBGENEk5G/fk7YZiHEZ2abgMIYz3762Yt05kJaJNnmtMfzYBMGUwEctL3uNN9zPUYIEQHQDaDK53ezQbLnfQDGTlAynIi2EtEmIrojC+1zw2+bl5rmgFeIaFqS3800vs9rmuFmAHjP9nY++tkPXteVr35OFnU8CwB/IqImIno4T23yYjER7SCit4mo3nyv4PuZiMpgbATW2t5Oup+HzGMvBwNE9BcAFgK4zvZ2rRDiEBHVAXiPiHYKIfblp4UO3gDwohCij4gegaGNfSPPbfLL3QBeEUJEbe8Vaj8PWojoBhjC4Brb29eY/TwBwDtE9JW5A84322CMgbNE9G0A6wDMym+TfHM7gE+FEHYtIul+HmyawSEA02yvq833XI8hoiCACgBdPr+bDXydl4huAvAzAN8RQvTJ94UQh8z/2wB8AODybDbWJGGbhRBdtnY+C6DB73ezRDLnvRuKSp2nfvaD13Xlq599QURfgzEulgghuuT7tn4+DuA15MZUmxAhxBkhxFnz77cAlBDROBR4P5vEG8/++zkXjpAMOlSCMBxlMzDgzKlXjvkRnA7k35t/18PpQG5DbhzIftp8OQwn1Szl/UoAw8y/xwHYgxw4r3y2ebLt7+8C2GT+PRbAfrPtlebfYwuhzeZxl8BwrlG++9l2/unwdmz+b3A6kD/PZz/7bHMNDJ/cVcr7IwGU2/7+DMAtBdLmSXJMwFg4O8w+9zWu8tFm8/MKGH6Fken2c04uKMOd820Au83F82fmeyth7KgBYDiAl83B+DmAOtt3f2Z+rxXArQXU5g0AjgHYbv5bb75/FYCd5gDcCeCBAmrzfwfQYrbtfQCX2L77Q7P/9wL4T4XSZvP1CgCPK9/LZz+/COAIgH4Y9ugHADwK4FHzcwLwpHlNOwEsLIB+TtTmZwGEbON5q/l+ndnHO8yx87MCavOPbeN5E2yCzG1cFUKbzWPuhxEYY/9eSv3M5SgYhmGYQeczYBiGYbIACwOGYRiGhQHDMAzDwoBhGIYBCwOGYZiCIVFxOpfjv09GkcsWIlqd1rk5mohhGKYwIKJrAZyFUY9qXoJjZwH4PYBvCCFCRDRBGElmKcGaAcMwTIEgXIrTEdFMIvqDWWfoYyK6xPzoIQBPCrMoYDqCAGBhwDAMU+isAvBfhBANAP4OwP9rvn8xgIuJ6FOzwGJaVWu5UB3DMEyBQkSjYGTIv2xU4gdglNQBjPV7FoxnMVQD+IiILhVCnE7lXCwMGIZhChcNwGkhxHyXzzphPDioH8B+ItoNQzhsSfVEDMMwTAEihDgDY6G/C7AegyofI7oOhlYAs8LqxTCK6qUECwOGYZgCgYheBLARwGwi6iSiBwDcB+ABIpKF5+ST1v4IoIuIdsEoFvn3wlYuPOlzc2gpwzAMw5oBwzAMw8KAYRiGYWHAMAzDgIUBwzAMAxYGDMMwDFgYMAzDMGBhwDAMwwD4/wHlQQF6Kvi4MAAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(torch.arange(0,len(conc2)), conc2, '.')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[<matplotlib.lines.Line2D at 0x7ff7759c01f0>]"
+ ]
+ },
+ "execution_count": 61,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAdGElEQVR4nO3dfZRV9X3v8fdnhgdFjQw4UgIokBAbSG9RTxCbrNbGBpHmFpumCYmr0lxzSXrNuvEm6zaQ9ObBNGuZ2yamrpsYqNqQLA1yNVFqSA1RkrTrCnJGUZ5CGBFkEGGE4UFRYGa+94/zm/EwzJw5A/PE2Z/XWmfN3t/fb5/925vNd+/z20+KCMzMLBuqBroBZmbWf5z0zcwyxEnfzCxDnPTNzDLESd/MLEOGDHQDSrnoooti4sSJA90MM7OzSl1d3SsRUdtZ2aBO+hMnTiSfzw90M8zMziqSdnZV5u4dM7MMKTvpS6qW9IykR9P4JElrJdVLekDSsBQfnsbrU/nEou9YlOJbJV3X60tjZmYl9eRI/zPAlqLxbwB3RMTbgSbg5hS/GWhK8TtSPSRNBeYB04DZwHclVZ9Z883MrCfKSvqSxgN/CtydxgW8D3gwVVkK3JCG56ZxUvm1qf5cYFlEHIuIF4B6YEYvLIOZmZWp3CP9bwN/C7Sm8dHAwYhoTuMNwLg0PA7YBZDKD6X67fFOpmknaYGkvKR8Y2Nj+UtiZmbd6jbpS/oAsC8i6vqhPUTEkojIRUSutrbTK47MzOw0lXPJ5nuAP5M0BzgHeAvwT8BISUPS0fx4YHeqvxuYADRIGgJcCOwvircpnqbX1e1sYs32/cycPJorL63pq9mYmZ1Vuj3Sj4hFETE+IiZSOBH7RETcCKwGPpSqzQceScMr0jip/IkoPL95BTAvXd0zCZgCPNVrS1KkbmcTN969hm/+fCs33r2Gup1NfTEbM7Ozzplcp/954LOS6in02d+T4vcAo1P8s8BCgIjYBCwHNgP/BtwSES1nMP8urdm+n+PNrbQGnGhuZc32/X0xGzOzs06P7siNiF8Cv0zD2+nk6puIeAP4yy6m/zrw9Z42sqdmTh7NsCFVnGhuZeiQKmZOHt3XszQzOysM6scwnK4rL63hvk/MdJ++mVkHFZn0oZD4nezNzE7mZ++YmWWIk76ZWYY46ZuZZYiTvplZhjjpm5lliJO+mVmGOOmbmWWIk76ZWYY46ZuZZYiTvplZhjjpm5lliJO+mVmGOOmbmWWIk76ZWYaU82L0cyQ9JelZSZskfTXFvy/pBUnr02d6ikvSnZLqJT0n6Yqi75ovaVv6zO9ilmZm1kfKeZ7+MeB9EfGqpKHAf0j6WSr7nxHxYIf611N4/+0U4CrgLuAqSaOALwM5IIA6SSsiwi+wNTPrJ+W8GD0i4tU0OjR9osQkc4EfpOnWACMljQWuA1ZFxIGU6FcBs8+s+WZm1hNl9elLqpa0HthHIXGvTUVfT104d0ganmLjgF1FkzekWFfxjvNaICkvKd/Y2NizpTEzs5LKSvoR0RIR04HxwAxJ7wIWAb8LvBsYBXy+NxoUEUsiIhcRudra2t74SjMzS3p09U5EHARWA7MjYk/qwjkG/AswI1XbDUwommx8inUVNzOzflLO1Tu1kkam4XOB9wO/Sf30SBJwA7AxTbICuCldxTMTOBQRe4DHgFmSaiTVALNSzMzM+kk5V++MBZZKqqawk1geEY9KekJSLSBgPfCpVH8lMAeoB44CHweIiAOSvgasS/Vui4gDvbYkZmbWLUWUuhBnYOVyucjn8wPdDDOzs4qkuojIdVbmO3LNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ8p5XeI5kp6S9KykTZK+muKTJK2VVC/pAUnDUnx4Gq9P5ROLvmtRim+VdF2fLZWZmXWqnCP9Y8D7IuL3genA7PTu228Ad0TE24Em4OZU/2agKcXvSPWQNBWYB0wDZgPfTa9gNDOzftJt0o+CV9Po0PQJ4H3Agym+lMLL0QHmpnFS+bXp5elzgWURcSwiXqDwDt0ZvbEQZmZWnrL69CVVS1oP7ANWAc8DByOiOVVpAMal4XHALoBUfggYXRzvZJrieS2QlJeUb2xs7PECmZlZ18pK+hHREhHTgfEUjs5/t68aFBFLIiIXEbna2tq+mo2ZWSb16OqdiDgIrAauBkZKGpKKxgO70/BuYAJAKr8Q2F8c72QaMzPrB+VcvVMraWQaPhd4P7CFQvL/UKo2H3gkDa9I46TyJyIiUnxeurpnEjAFeKqXlsPMzMowpPsqjAWWpittqoDlEfGopM3AMkl/DzwD3JPq3wP8UFI9cIDCFTtExCZJy4HNQDNwS0S09O7imJlZKSochA9OuVwu8vn8QDfDzOysIqkuInKdlfmOXDOzDHHSNzPLECd9M7MMcdI3M8sQJ30zswxx0jczyxAnfTOzDHHSNzPLECd9M7MMcdI3M8sQJ30zswxx0jczyxAnfTOzDHHSNzPLECd9M7MMcdI3M8uQcl6XOEHSakmbJW2S9JkU/4qk3ZLWp8+comkWSaqXtFXSdUXx2SlWL2lh3yySmZl1pZzXJTYDn4uIpyVdANRJWpXK7oiIfyyuLGkqhVckTgPeCvxC0jtS8XcovGO3AVgnaUVEbO6NBTEzs+51m/QjYg+wJw0fkbQFGFdikrnAsog4BryQ3pU7I5XVR8R2AEnLUl0nfTOzftKjPn1JE4HLgbUp9GlJz0m6V1JNio0DdhVN1pBiXcXNzKyflJ30JZ0PPATcGhGHgbuAtwHTKfwS+GZvNEjSAkl5SfnGxsbe+EozM0vKSvqShlJI+PdFxI8BImJvRLRERCvwz7zZhbMbmFA0+fgU6yp+kohYEhG5iMjV1tb2dHnMzKyEcq7eEXAPsCUivlUUH1tU7c+BjWl4BTBP0nBJk4ApwFPAOmCKpEmShlE42buidxbDzMzKUc7VO+8B/grYIGl9in0B+Kik6UAAO4BPAkTEJknLKZygbQZuiYgWAEmfBh4DqoF7I2JTry2JmZl1SxEx0G3oUi6Xi3w+P9DNMDM7q0iqi4hcZ2W+I9fMLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDynlH7gRJqyVtlrRJ0mdSfJSkVZK2pb81KS5Jd0qql/ScpCuKvmt+qr9N0vy+WywzM+tMOUf6zcDnImIqMBO4RdJUYCHweERMAR5P4wDXU3gZ+hRgAXAXFHYSwJeBq4AZwJfbdhRmZtY/uk36EbEnIp5Ow0eALcA4YC6wNFVbCtyQhucCP4iCNcBISWOB64BVEXEgIpqAVcDs3lwYMzMrrUd9+pImApcDa4ExEbEnFb0MjEnD44BdRZM1pFhX8Y7zWCApLynf2NjYk+aZmVk3yk76ks4HHgJujYjDxWUREUD0RoMiYklE5CIiV1tb2xtfaWZmSVlJX9JQCgn/voj4cQrvTd02pL/7Unw3MKFo8vEp1lXczMz6STlX7wi4B9gSEd8qKloBtF2BMx94pCh+U7qKZyZwKHUDPQbMklSTTuDOSjEzM+snQ8qo8x7gr4ANktan2BeA24Hlkm4GdgIfTmUrgTlAPXAU+DhARByQ9DVgXap3W0Qc6I2FMDOz8qjQHT845XK5yOfzA90MM7OziqS6iMh1VuY7cs3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDnPTNzDLESd/MLEOc9M3MMsRJ38wsQ5z0zcwyxEnfzCxDynld4r2S9knaWBT7iqTdktanz5yiskWS6iVtlXRdUXx2itVLWtj7i2JmZt0p50j/+8DsTuJ3RMT09FkJIGkqMA+Ylqb5rqRqSdXAd4DrganAR1NdMzPrR92+Izcifi1pYpnfNxdYFhHHgBck1QMzUll9RGwHkLQs1d3c8yabmdnpOpM+/U9Lei51/9Sk2DhgV1GdhhTrKn4KSQsk5SXlGxsbz6B5ZmbW0ekm/buAtwHTgT3AN3urQRGxJCJyEZGrra3tra81MzPK6N7pTETsbRuW9M/Ao2l0NzChqOr4FKNE3MzM+slpHelLGls0+udA25U9K4B5koZLmgRMAZ4C1gFTJE2SNIzCyd4Vp99sMzM7Hd0e6Uv6EXANcJGkBuDLwDWSpgMB7AA+CRARmyQtp3CCthm4JSJa0vd8GngMqAbujYhNvb0wZmZWmiJioNvQpVwuF/l8fqCbYWZ2VpFUFxG5zsp8R66ZWYY46ZuZZYiTvplZhjjpm5lliJO+mVmGOOmbmWWIk76ZWYY46ZuZZYiTvplZhjjpm5lliJO+mVmGOOmbmWWIk76ZWYY46ZuZZYiTvplZhjjpm5llSLdJX9K9kvZJ2lgUGyVplaRt6W9NikvSnZLqJT0n6Yqiaean+tskze+bxTEzs1LKOdL/PjC7Q2wh8HhETAEeT+MA11N4L+4UYAFwFxR2EhRes3gVMAP4ctuOwszM+k+3ST8ifg0c6BCeCyxNw0uBG4riP4iCNcDI9BL164BVEXEgIpqAVZy6IzEzsz52un36YyJiTxp+GRiThscBu4rqNaRYV/FTSFogKS8p39jYeJrNMzOzzpzxidwovFm9196uHhFLIiIXEbna2tre+lozM+P0k/7e1G1D+rsvxXcDE4rqjU+xruJmZtaPTjfprwDarsCZDzxSFL8pXcUzEziUuoEeA2ZJqkkncGelmJmZ9aMh3VWQ9CPgGuAiSQ0UrsK5HVgu6WZgJ/DhVH0lMAeoB44CHweIiAOSvgasS/Vui4iOJ4fNzKyPqdAlPzjlcrnI5/OnNW3dzibWbN/PzMmjufJSXx1qZtkhqS4icp2VdXukfzaq29nEjXev4XhzK8OGVHHfJ2Y68ZuZUaGPYVizfT/Hm1tpDTjR3Mqa7fsHuklmZoNCRSb9mZNHM2xIFdWCoUOqmDl59EA3ycxsUKjI7p0rL63hvk/MdJ++mVkHFZn0oZD4nezNzE5Wkd07ZmbWuYpN+nU7m/jO6nrqdjYNdFPMzAaNiuze8SWbZmadq8gj/eJLNo+daOXbv/itj/jNzKjQI/2aEcNoTTcaB/Af215h7fb9/GVuAh+8YnynR/2+g9fMsqAik/7Glw6dNB7A8Zbg/rUv8tDTDad097g7yMyyoiK7d9RFPOj8Dl3fwWtmWVGRSX/aWy88aXzGxJqSd+j6Dl4zy4qK7N5pOnq8fVjAH112MZ+//p1d9tn7Dl4zy4qKTPo1I4a1D0eH8a74Dl4zy4KKTPodT+Su3rqP2x7d5BO1ZpZ5Z9SnL2mHpA2S1kvKp9goSaskbUt/a1Jcku6UVC/pOUlX9MYCdNquDuMbGg76RK2ZGb1zIvePI2J60VtaFgKPR8QU4PE0DnA9MCV9FgB39cK8O/XBK8YzbMibi7b38DFaA6p8otbMMq4vrt6ZCyxNw0uBG4riP4iCNcBISWP7YP5ceWkNX/nP05g4egSi0K9fBbzn7Re1d+342TxmlkVn2qcfwM8lBbA4IpYAYyJiTyp/GRiThscBu4qmbUixPfSyup1N3PboJo6daC0kfMGwIVXc+ifvaE/4vhnLzLLoTJP+eyNit6SLgVWSflNcGBGRdghlk7SAQvcPl1xyyWk1as32/bxxorV9fHLt+XzjL/5Te2Lv7GYsJ30zy4Iz6t6JiN3p7z7gJ8AMYG9bt036uy9V3w1MKJp8fIp1/M4lEZGLiFxtbe1ptWvb3iMnjdfve5XFv3q+vSvHN2OZWVaddtKXdJ6kC9qGgVnARmAFMD9Vmw88koZXADelq3hmAoeKuoF61fpdB0+J/XzzXm68ew11O5vab8b67KzL3LXTD3z+xGzwOJPunTHATyS1fc/9EfFvktYByyXdDOwEPpzqrwTmAPXAUeDjZzDvki4ZNYId+4+eEn/jRCu33FfHf7/2HXzsqkuc7PuBz5+YDS6nnfQjYjvw+53E9wPXdhIP4JbTnV9PHHr9RJdlLx8+xhd+soEX97/GBecO9WMX+pjPn5gNLhV5R+7FbzkHOFSyzuJfbwegukrcNvddXPY7F/jZO32g7fzJieZWnz8xGwQqMun/8WUXs2rz3pJ12i4pam4N/tfDG6iurqK5xV0Qvc0PszMbXCoy6TcdPd5+U1Y5WgNam1tPet6+k1Pv8cPsyuc3uFlfq8ik39MuhKoqABER7oLoA05k5fFJb+sPFZn0f/jkjrKP8gFaWgGCKsFfXz3R/9G60ZMk7kRWPp/0tv5QkW/O+umG0pf/VwnefvH5p8RbAxb/+/ayryevhOvPb1+5hWv+YTW3r9xSVv26nU3MW/Ik//DYVuYtebLbZferKMvnmwatP1Tkkf6JltLH+a0Bz+97tdOyCPjGz7aw/FN/UPI76nY28ZElT9LcEgypFg8suPqsOyq7feUWvpeuYmr7u3DOO0tOs/hXz7ev3xMtwecfeu6kR1x0VJy4gp53vWWJT3pbf6jII/1ylNot5Ms4cl/8q+dpTsmvuSVY/Kvne6ll/ef+p14sOd6ZtTtOPlKv3/cqH1n8JPevfXPatl9A9699kb/7yQZa08pujULXWyX8QuotXhfW3yrySP9MtXayR+jYj/3UjgMnle89/EY/te70dVyGo8ebTypvG79/7Yv8bOMern/X2FPuXzjyevMp39vcGnwx3fB2yejz+NIjG2lpjU53rA+vf4mVG1/25bGcer7jSx+Yxlf+dVP7PQ0/+q8zAdrXf/FwVteZnTkn/S68bdFPqb1gODMnj2b3wddZt6NwJFYtmHzReRw8evJdv/tfPdY+XLezie/96nn2HX6Dj7z7Ej521ek9LfR03LrsGX7520aueUct3553+Ult6nhCtbn15GmbW+Gme9by622vAPDv6W+bC4ZXd7pDhMIvp7Yuou4cTzM+XtTHX5zMBnL99bXiHe+Pn25of/z3ieZWHlj34knr5vMPPsuLTa/T3NLKkCqB5J2lnTEn/S60ROGRDQ+vf+mU+LbG106p33DwDaZ/9THOO2coew6+3p4cn23YAMDHrrqk5FUvHcvaEt8Lja8yufZ8PvlHb+v2P/mty55pb2/b32/Pu7zwfoF/3dT+uOm2ZxB15tcdEn2xI8daSs6/p1qj8ETUO37xW5pbguoquPKSGtbtaGr/lfBsQ9ePzKjb2cRDTzcgYNpbL6Tp6PFePwou/neB7o+0u6oP8NDTDSzP76IlLStS+3JWV1e1J/w29UXb2fGWoK1T8tiJVh56uqF9O/HRv/WECo/EGZxyuVzk8/keTzdx4U/7oDWnb2i1GD1iGC8fefPXwKf+cDIL57yzPXE9WNfQfhT3pQ9M4+8e3nDSUXVV+oUxufZ8rrnsYja+dIgnNu/lldeOMb5mBN/88HTmLXnylJPYPblJ7Wwg4Lxh1ahKHHnj1K4mgCm157Hqc9e0j7clxpoRw1i9dV/JXxAdk/Zf3PX/2suqVNhRVQu+dsPvnbSTuXXZM6zcsIfjLYFS3eJ/iu7+HS4YXt3jneqlo0aw88CbDxacNXUMn/yjtwG07ww/eMX4bg8wKkV3y9VWfuT1E2zac5jr3zW2on5FFpNUV/QK25PLnPStUl06agT7jrzBG6kLpaNRI4YyYdQIPvLuS3hx/2s8vH43ew8f6/FOcli10pH44DRjYiEBbn/lNY4eb+GNEy1Eemf0te98c0fRtmMs9YupnB1GV3WKd77lzKNmxDA2vnSI+r1HONbcyqSLzuOFV17jeHMrLx44yrHmVt711rfw/mm/w7a9R1jx7EvtO+V3jLmAlw69zmvHWhh13lA+ePl4vv/kjpNergRQXQWXjbmAr93we+3roPjXdsedZ7k7lp7uUHt7R+ykb2Y9NmrEUA6/cYIqiWqJE61Bc/r5ObRaXD15NOt2HGjfqY48dwgBHCo62f+pP5zMk9v3s+vAUQ4cPfXptzMm1rD74Ovsf+34KQl5sOm4cz93aBXDhlZxojmogpN+fQp4d1o2JKaNfUt7F23bzqR+7xEOvHacodVV/Hbfq7S2BtVV4hPvncThY81d/lIrh5O+mdlZ6qG/+YMeJ/5SST+z1+mbmZ0NPrbkyV79Pid9M7NB7Fgvny/q96QvabakrZLqJS3s7/mbmWVZvyZ9SdXAd4DrganARyVN7c82mJllWX8f6c8A6iNie0QcB5YBc/u5DWZmmdXfSX8csKtovCHF2klaICkvKd/Y2NivjTMzq3SD7kRuRCyJiFxE5Gprawe6OWZmFaW/k/5uYELR+PgU61U7bv/T3v5KM7MB0dv5rL8fuLYOmCJpEoVkPw/4WF/MyInfzOxU/Zr0I6JZ0qeBx4Bq4N6I2NSfbTAzy7J+f7RyRKwEVvb3fM3MbBCeyDUzs77jpG9mliFO+mZmGeKkb2aWIYP6efqSGoGdZ/AVFwFdv/TVvH5K8/opzeuntIFcP5dGRKd3tw7qpH+mJOW7epGAef10x+unNK+f0gbr+nH3jplZhjjpm5llSKUn/SUD3YBBzuunNK+f0rx+ShuU66ei+/TNzOxklX6kb2ZmRZz0zcwypCKTfqW/fF3SBEmrJW2WtEnSZ1J8lKRVkralvzUpLkl3pvXxnKQrir5rfqq/TdL8oviVkjakae6UpFLzGGwkVUt6RtKjaXySpLVpeR6QNCzFh6fx+lQ+seg7FqX4VknXFcU73b66msdgJGmkpAcl/UbSFklXe/t5k6T/kf5vbZT0I0nnVMw2FBEV9aHwyObngcnAMOBZYOpAt6uXl3EscEUavgD4LYUXzf9vYGGKLwS+kYbnAD8DBMwE1qb4KGB7+luThmtS2VOprtK016d4p/MYbB/gs8D9wKNpfDkwLw1/D/ibNPzfgO+l4XnAA2l4atp2hgOT0jZVXWr76moeg/EDLAU+kYaHASO9/bSvm3HAC8C5Rf+uf10p29CAr+A++Ae7GnisaHwRsGig29XHy/wI8H5gKzA2xcYCW9PwYuCjRfW3pvKPAouL4otTbCzwm6J4e72u5jGYPhTeyPY48D7g0ZR4XgGGdNxGKLzb4eo0PCTVU8ftpq1eV9tXqXkMtg9wYUpq6hD39hPtSX8XhZ3ZkLQNXVcp21Aldu90+/L1SpJ+Sl4OrAXGRMSeVPQyMCYNd7VOSsUbOolTYh6DybeBvwVa0/ho4GBENKfx4uVpXwep/FCq39N1Vmoeg80koBH4l9QFdrek8/D2A0BE7Ab+EXgR2ENhm6ijQrahSkz6mSHpfOAh4NaIOFxcFoVDhT69Hrc/5tFTkj4A7IuIuoFuyyA2BLgCuCsiLgdeo9DV0i6r2w9AOs8wl8LO8a3AecDsAW1UL6rEpN8vL18faJKGUkj490XEj1N4r6SxqXwssC/Fu1onpeLjO4mXmsdg8R7gzyTtAJZR6OL5J2CkpLY3xRUvT/s6SOUXAvvp+TrbX2Ieg00D0BARa9P4gxR2At5+Cv4EeCEiGiPiBPBjCttVRWxDlZj021++ns58zwNWDHCbelW6EuIeYEtEfKuoaAXQdgXFfAp9/W3xm9JVGDOBQ+kn9mPALEk16ehmFoU+xD3AYUkz07xu6vBdnc1jUIiIRRExPiImUvi3fyIibgRWAx9K1Tqum7bl+VCqHyk+L12ZMQmYQuHkZKfbV5qmq3kMKhHxMrBL0mUpdC2wGW8/bV4EZkoakdrftn4qYxsa6JMmfXQiZg6FK1qeB7440O3pg+V7L4Wfxc8B69NnDoU+wceBbcAvgFGpvoDvpPWxAcgVfdd/AerT5+NF8RywMU3zf3jz7u1O5zEYP8A1vHn1zmQK/+Hqgf8LDE/xc9J4fSqfXDT9F9PybyVdfVJq++pqHoPxA0wH8mkbepjC1Tfeft5s/1eB36Rl+CGFK3AqYhvyYxjMzDKkErt3zMysC076ZmYZ4qRvZpYhTvpmZhnipG9mliFO+mZmGeKkb2aWIf8fimFBHijHfowAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(torch.arange(0,len(flat_fft2)), flat_fft2.abs(), '.')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 62,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "topk = torch.topk(\n",
+ " flat_fft2.abs(), round(0.1*len(flat_fft2)), dim=0, sorted=False\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 63,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "torch.return_types.topk(\n",
+ "values=tensor([ 75.7603, 695.2839, 721.5375, ..., 68.1649, 68.1649, 68.1649]),\n",
+ "indices=tensor([294037, 1, 2, ..., 241565, 434039, 328013]))"
+ ]
+ },
+ "execution_count": 63,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "topk"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 64,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([294037, 1, 2, ..., 241565, 434039, 328013])"
+ ]
+ },
+ "execution_count": 64,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "topk.indices"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 65,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "top10 = torch.zeros(len(flat_fft2), dtype = torch.cfloat)\n",
+ "top10[topk.indices] = flat_fft2[topk.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 66,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "84502"
+ ]
+ },
+ "execution_count": 66,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(topk.indices)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 67,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse_top10 = fft.irfft(top10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 68,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(34.8182)"
+ ]
+ },
+ "execution_count": 68,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_top10 - conc2, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 69,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 69,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc2, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 70,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(30254.6758)"
+ ]
+ },
+ "execution_count": 70,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(reverse_top10 - conc2, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 71,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " conc2.abs(), round(0.1*len(conc2)), dim=0, sorted=False\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 72,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "169005"
+ ]
+ },
+ "execution_count": 72,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(topk_og.indices)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 73,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 1, 2, 3, 4, 0, 6, 7, 8, 9, 10, 11, 12, 13, 16, 18, 19, 20, 21,\n",
+ " 22, 23, 24, 25, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 38, 40, 41, 42,\n",
+ " 44, 45, 46, 47, 48, 49, 39])"
+ ]
+ },
+ "execution_count": 73,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "topk_og.indices[topk_og.indices<50]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 74,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "top10_og = torch.zeros(len(conc2))\n",
+ "top10_og[topk_og.indices] = conc2[topk_og.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 75,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(15.5541)"
+ ]
+ },
+ "execution_count": 75,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(top10_og - conc2, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 76,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(15858.2695)"
+ ]
+ },
+ "execution_count": 76,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(top10_og - conc2, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 77,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0058, -0.0441, -0.0381, -0.0493, -0.0240, 0.0190, -0.0132, -0.0221,\n",
+ " -0.0472, 0.0077, 0.0236, 0.0183, 0.0231, 0.0014, -0.0245, -0.0085,\n",
+ " 0.0035, -0.0036, 0.0150, 0.0107, 0.0123, 0.0039, 0.0003, -0.0320,\n",
+ " -0.0093, 0.0632, 0.0360, 0.0200, -0.0248, 0.0029, -0.0011, -0.0193,\n",
+ " 0.0221, 0.0056, -0.0091, -0.0008, 0.0329, 0.0133, -0.0078, -0.0061,\n",
+ " -0.0372, -0.0354, -0.0238, -0.0028, 0.0145, -0.0121, -0.0517, -0.0468,\n",
+ " -0.0123, -0.0132])"
+ ]
+ },
+ "execution_count": 77,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "reverse_top10[10000:10050]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 78,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0155, 0.0128, -0.0386, 0.0186, 0.0166, 0.0259, -0.0200, -0.0033,\n",
+ " -0.0399, 0.0214, 0.0106, 0.0197, -0.0182, -0.0191, -0.0370, 0.0159,\n",
+ " 0.0071, -0.0321, -0.0166, -0.0082, 0.0090, 0.0291, 0.0117, 0.0011,\n",
+ " 0.0066, 0.0163, 0.0237, 0.0092, -0.0029, -0.0209, -0.0207, 0.0039,\n",
+ " 0.0065, 0.0057, 0.0316, -0.0262, -0.0342, -0.0115, 0.0149, -0.0175,\n",
+ " -0.0568, -0.0135, -0.0503, -0.0252, 0.0148, -0.0429, -0.0424, -0.0182,\n",
+ " -0.0002, -0.0341])"
+ ]
+ },
+ "execution_count": 78,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "conc2 [10000:10050]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 79,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0000, 0.0000, -0.0386, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " -0.0399, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, -0.0370, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " -0.0568, 0.0000, -0.0503, 0.0000, 0.0000, -0.0429, -0.0424, 0.0000,\n",
+ " 0.0000, 0.0000])"
+ ]
+ },
+ "execution_count": 79,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "top10_og[10000:10050]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 80,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1440, -0.0482, 0.2070, ..., 0.0011, 0.0177, -0.0218])"
+ ]
+ },
+ "execution_count": 80,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "conc2[0:10000]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 81,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reverse_top10fft = reverse_top10"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 82,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACSL0lEQVR4nO2dd3hjZ5m371fdsizJ3dM9vU8myaQnMKQTQghtISSQLCVLZ1nY3SzLtwQIEFggIRD6QkIvgRRqSO+ZZJLMZPp4+njGvUiWZPX3++M9R82SLNmyPWOf+7p82To60nklS+c5T/s9QkqJgYGBgcHMxTTVCzAwMDAwmFoMQ2BgYGAwwzEMgYGBgcEMxzAEBgYGBjMcwxAYGBgYzHAMQ2BgYGAwwzEMgYHBSY4QYqMQon2q12Fw8mIYAoMpRQhxSAgxLIQICCG6hBB3CSFcU72uYgghWoUQUghhqdDzOYQQg0KIC/Pcd5sQ4p5KHMfAoBCGITA4EXijlNIFnAZsAD5bzoOF4qT5LOcaECllGPgt8J6c/czANcDdk7c6g5nISfPlMZj+SCmPAX8D1gghaoUQfxZC9AghBrS/5+r7CiEeF0J8SQjxDBACFgkh/lkIsUsIMSSEOCCE+JeM/TcKIdqFEP8hhOgWQnQIIa4WQlwhhNgrhOgXQnwmY3+TEOImIcR+IUSfEOJ3Qog67e4ntd+DmidzjvaY92rHHxBCPCiEWJDxfFII8REhRBvQlufl3w28VQjhzNh2Geo7+rdiry0X7VhLMm7fJYS4JeP2lUKILZoX8qwQYl3Gff8phDimHWePEOKiQscxmD4YhsDghEEIMQ+4AngF9dn8KbAAmA8MA9/Jeci7gRuBGuAw0A1cCbiBfwZuE0KclrF/C+AA5gD/A/wIuA44HbgA+H9CiIXavh8DrgZeC8wGBoA7tfteo/32SildUsrnhBBvAj4DvAVoBJ4Cfp2z3quBs4BVua9dSvks0KE9PvP1/UpKGS/htZWEEOJU4CfAvwD1wA+AB4QQdiHEcuCjwBlSyhqUITpU7jEMTkKklMaP8TNlP6gTTQAYRJ3MvwtU5dlvPTCQcftx4AujPPd9wCe0vzeijIlZu10DSOCsjP1fAq7W/t4FXJRx3ywgBliAVu2xloz7/wa8L+O2CeWpLNBuS+DCUdb7WeAf2t9u7fGnlvja2jPuk8CSjNt3Abdof38P+GLOc+1BGbwlKINzMWCd6s+G8TN5P4ZHYHAicLWU0iulXCCl/LCUclgI4RRC/EAIcVgI4UeFY7xa3FznaOaTCCFeL4R4XgvzDKK8i4aMXfqklAnt72Htd1fG/cOAnqheANyrhU8GUYYhATQXeA0LgG9l7N8PCJT3kXe9efg58DohxGzgbcB+KeUrJb62UlkAfEpfp/Zc84DZUsp9wL8CNwPdQojfaGsxmOYYhsDgROVTwHLUFbubdDhGZOyTks4VQtiBPwBfB5qllF7grzn7l8NR4PWagdJ/HFLlMfJJ9h4F/iVn/yqpQj4j1psPKeVhVEjpOlRY6O4xvrYQkJlraMlZ55dy1umUUv5aW8OvpJTnowyGBL5abM0G0wPDEBicqNSgrtAHtSTt50bZ3wbYgR4gLoR4PXDpOI7/feBLesJXCNGo5QHQjpEEFuXs/19CiNXa/h4hxNvHcNy7UXH684BfatvKfW1bgHcJIcxCiMtRYR+dHwEfFEKcpVVbVQsh3iCEqBFCLBdCXKgZnjDq/U+O4TUYnGQYhsDgROV2oAroBZ4H/l5sZynlEPBx4HeoxO67gAfGcfxvaY//hxBiSFvDWdqxQsCXgGe08MrZUsp7UVfPv9FCWduB14/huH8A6oBHpJQdY3xtnwDeiMq7XIvKJ6A912bgA6jE+wCwD7hBu9sO3Ip6zzuBJuC/xvAaDE4yhJTGYBoDAwODmYzhERgYGBjMcAxDYGBgYDDDMQyBgYGBwQzHMAQGBgYGM5yKqCdONg0NDbK1tXWql2FgYGBwUvHSSy/1Sikbc7eflIagtbWVzZs3T/UyDAwMDE4qhBCH8203QkMGBgYGMxzDEBgYGBjMcAxDYGBgYDDDMQyBgYGBwQzHMAQGBgYGMxzDEBgYGBjMcAxDYGBgYDDDqYghEEJcrg263ieEuCnP/XYhxG+1+zcJIVq17ZcIIV4SQmzTfl9YifUYGBhMMD174OCTU70KgwoxbkOgjQ68E6W9vgq4RgiRO5z7fah5s0uA20hPPeoF3iilXAtcjxrVZ2BgcKLz+K3wwMemehUGFaISHsGZwD4p5QEpZRT4DfCmnH3ehDZ2D7gHuEgIIaSUr0gpj2vbdwBV2nQkAwODE5lQH0SGpnoVBhWiEoZgDtlDudvJHtidtY+UMg74gPqcfd4KvCyljOQ7iBDiRiHEZiHE5p6engos28DAYMyEByEamupVGFSIEyJZrM15/SrwL4X2kVL+UEq5QUq5obFxhGaSgYHBZDI8APFhSCameiUGFaAShuAYMC/j9lxtW959hBAWwAP0abfnAvcC75FS7q/AeqY30SD87Gro3j3VKzGYyQwPqt8xwyuYDlTCELwILBVCLBRC2IB3MnKw9gOoZDDA24BHpZRSCOEF/gLcJKV8pgJrmf70tsGBx+DopqleicFMJRGHiF/9bYSHJo1n9vXy5b/uYjhaeS9s3IZAi/l/FHgQ2AX8Tkq5QwjxBSHEVdpu/wfUCyH2Af8G6CWmHwWWAP8jhNii/TSNd03TmmCv+h0NTO06DGYuYV/6b+NzOGm8cLCfHz11AKtZVPy5KzKPQEr5V+CvOdv+J+PvMPD2PI+7BbilEmuYMYQ0QxAxvoAGU0R4MP23ERqaNHoCEeqrbVjMlU/tnhDJYoMySHkERumewRQxPJD+OxqcunXMMHqGIjS4Jqa63jAEJxu6R2B8AQ2mCsMQTAk9QxEaawxDYAAQ1HoojNCQwVShVwyBYQgmkZ6hCI2GR2AAQLBP/TaSdAZTRaZHYOQIJgUpJb2BifMITsrh9TOaVLLYyBHoPLyzi5eODFBtM+O0Wai2p38vbaphXp1zqpc4vcgKDRkXJJPBUCROJJ40DIGBhh4aMr6AKT73wA6ODQ6P2H6OaQdnVHXyyc9+AyEqX3I3YwkPgjCDTBh9BJNEz5BS3pmoZLFhCE42UqEhIzYLEIjEOTY4zKcvXcaNr1lMKBonFE0Qisax/eLrNPq20+X7Ei3eqqle6vRheABqWsB/zPgcThK6ITCSxQYQj6TLRo1kMQD7utX7sLS5BpvFhNdpY7a3iiVeE3MD26gSUdoOtE3xKqcZw4PgrANLFcQMQzAZGIbAII3eQ2B1GqEhjb1dyjAubXJl33H4WUzJGADdhwxdpooyPABVtWBzGh7BJJEyBEbVkEEqP1DbqgyBlFO6nBOBfd0BbBYTC+qrs+/Y/1jqz0DH3kle1TRneAAcXrBVGzmCSaI3EMFqFniqrBPy/IYhOJnQK4ZqW0EmjdI9lEewuNGF2ZSTDD7wOCw4nwQmGDg4JWs7mYnEEzy2u5v/uX87T+zNmf8RHlQegbXa8EwniZ6hCPXVdky5n/MKYSSLTyb0RHFtq/odDaqrshlMW1eA0xfUZm8c6oLuHXDxzQS6DlIbPIZvODZhV1PTBX84xmO7u/nHzi4e391NUFO57PCFee0ybQaIlBmhoWrjYmSS6JnAHgIwDMHJRWZoCFQvgasEsdaBw5CMQ/3iCVvaVBDUKoauOXNe9h0HHle/F72OxPaHmB/qYFeHn7MX5Q7FMwBIJCUf/dXLPLyri1hC0uCycdX62Vy6qoXvP7Gf3kDG0MBYCBJRqPIaOYJJpGcoQrPbMWHPbxiCk4lQL5gs4J6tbpfqlv/tPyHYDR94dOLWNgXoFUNLmmqy7zjwGDjroWUdjuYltHZs4Q/HZ44h2N8T4LHd3bz/gkUl7X+wN8Dftndy5bpZ3HBuK6fOr02F2u7fcoyXjwymd9blJapqweaC0MCI5zOoPD1DEdbM9kzY8xs5gpOJYC84G8CunfhKLSENdquBNtMsuaxXDC1rzqgYklIlihe+FkwmnM1L8YogB462T9EqJ58/vtzOLX/Zlao0GY09nepz9MHXLmZDa11WvqXeZc/2CPSu4qpao3ptkkgmJX3B6ISGhgxDcDIR6oPqBrBphqDUL2HYryZKDU+vq7e27gA2s4n5mRISPXsg0AmLNqrbtQsB8B+bOZVDgyFVNrunszQZkj1dQ5gELMktwUV1suoNekB6FoHDq0JDMy1HEOyDoy9M6iEHQlESSRWymygMQ3AyEexRIQ+79oUt1RDoYwUHD0/MuqaItq4hFjVWZw/qOKCVjS5+nfpdpwyBGDxEODYzBq3rhmB3p7+k/fd2DtFaX43Dah5xX7128ukLRNWGTI/A5pp55aPP3gE/ff2kXlT1BPRmsonLERiG4GQi2AvVjelKoVJDQ7pA3cChCVnWVLG3K8Cy5pz8wP7HoG4xeOer21pifZ7spK1rZoQxBofVSXt3iR7B3u4hljaP9AYg3cCkn4zShsCbDg1Ns5BjUXxHVeFFRp/KRDPRXcVgGIKTi1RoqAyPIBFLu+/TyBDoFUNZHcXxKBx6Ou0NANiqiTubWSC62HHcN/KJpiEDwdJDQ+FYgkO9QZbnGlQNXeQs7REMqt96+SgS4uHxLvnkwd+hfu97eNIOaRgCgzTxiArxODMMQSkeQaZc9TQyBJkaQynaX1TaN3p+QMNcv4iF5m52HC8tVHKy4xtWhmBv1xCJZPGr9f09AZISlrXkNwR6aKg30yMwWdRnUPdMZ1IJ6VCGIUgmJ+WQvQHDEBjo6DpD1fVgtijBr1LmFoczroIHpk+OIKUxlBnSOPAYCBO0XpC1r6hbyCJzDzs7ZoYhGAhF8TqtROJJDvcVP0nr72MhjyCdI9AMQXhQJYqFqJgheP5AH4Oh6LieY1KQUhkCVzMEuqBr26QctmcogsNqoto2ModTKQxDcLKgy0tUax2edldpX0A9UWypmnYegc1sYkFmxdCBx2HO6Sp+nUndQuqTvRzo6B31CnmiiSeSbDrQN2HPH4knCEUTnLWwDhg9PLS3K4DVLGhtyN+hbreYqXFY6M1MFldpndxW7b0fhyEIxxJc9+NN/OipA2N+jkkjPKjCYKdco263/WNijvPIF+C316Vu6rOKJ3KmhmEIThZ0j8DZoH7bqssLDTWv1hJd06NyZm9uxdDwIBx7CRa9buTOdaqxqiHWwaFRrpAnmj+/2sE7fvg8Lx3un5Dn92kVQ2e01mESoyeM93YOsajBhdVc+FTQmNlLkGkI9BDlOEpIewMR4knJ7o6TYOKenh+YtQ5mrYe2CcoTHH0Bdv8llZjvCUzcrGIdwxCcLKRCQ7ohqCktWRzWPIJZ61S1g//YxKxvkmnrDmTnBw49pYT4cvIDQKqXQCWMpzY8tOXoIAAP7+qekOcf0AxBi8dBa331qB7Bnq6hgvkBnXqXLcMQDKY9LpvuEYy9GktPhLZ1nwQVXXp+oGYWLL0E2l+YmDLSUL/6LB98Ekh7BBOJYQhOFkI5hsDuKm1usR4aalmrfk+D8FAwEqd9IKdiaP9jSg1z7hkjH6D1Eiw0d7Nzig2BfvzHdk+MIdBj7bVOG8tbakb2EsTSFT76+7i8QOmoToPLnt1HkPII9BzB2D0C3RAcHQgxHD3BvdWhTvW7ZhYsvVSdrPdPgGyL/l3XntswBAZpgprOkMOrbttc5XkELaeo39PAEOzvUa97WW6iuPV8sOTpvqyqBYeHU5z9U1pCmkxKdhz34bSZ2d05lHfO8njRPQJPlZXlLTUc7g+lu4IPPwtfmQN9+4H0VfjSAolinYas0NBg+jNoHX+yWM89SJn+v56wDB1Xv2tatFxUbeXDQ1KqMnGAfY8SiycYCMUmbFaxjmEIThb0rmI9YWR3lZgj0AxB0wo1cHwaVA7t7co5gQ0chv4D2f0DmQgBtQtZau1h53E/cooaoA73hwhGE1x39gIAHp0Ar8CnNZPVVttY0VKDlKQb6V75pQoPavMZ9nYWrxjSqXfZGAjFiMViEPGN9AjGMa4yUw9Jr2A6YRnq1DSWqsBkhsUXVr6MNOJX/6O6ReA7wmD7HmBiS0fBMAQnD6G+dKIYtOlQJVYNme1qf8/caeERtHUNZVcMpWSnNxZ+UN1CZiU66AtG6fKXJsZWaXRv5KpTZrOg3jkh4SHdI/BWWVne4ga0yqF4BHb9Se2klRTv6RrCYTUxL7PyKg/61ehgvyaDnjIE468a6gmEqXFYsJrFiZ8n8HeosJDO0kuVoGPn1sodQ/cG1r4dgOjeR4CJG1GpYxiCk4Vgbzo/AOUlix3qhEBt6/QwBN2B7IqhA4+pL2jjisIPql2IK9yBmQQ7O6YmPLT9mB+rWbC02cXrljfxzL7eisfFB0MxbGYTTpuZ+XVOHFaTqhza/6i6moeUIdjbNcTSppqR091y0MXO0obAq35bK5MjmOVxsLChmrZcj2D7H+DAE2N+7oozlGMIFl+kflcyPBTSqsnmbADvAmyHlZSF4REYKEI5hsDuKk3nJeIH+wloCNoehh9dqCQwymRv11C2Umb7Zph/Tjpslo+6RZhknNmilx3HpiZhvOO4j6VNNdgtZi5c0UQknuS5A70VPcZgKIrHaUUIgdkkWNZcw54uvzqp6p8DLW+0t6uwxlAmukcQGMjxCCw2MFnHXTXUWGNnaVNNtkew7R64573wzO1jfu6KM9RBp6zlv/64TYUXXY0w+1TY91DljpEqE6+HxRfi7XoeC3HDEBho6LMIdGyu0uYWR4bS8wtqFyiDUqpY3UTS9g9V969XYpRIKKoqXVJic5GA6o9oWln8gVrl0Jke35SUkEop2XHcz+rZ6mR81qI6nDZzxfMEg6EYtc70SM7lzTUcPN4Le/4Gq9+sCg7CPgZDKkQ2Wn4A1EwCgJA/xxDAuMdV9gaiNLjsLG12caRfqxw6/Bzc9yG1Q3jqkvtZJBMQ6OKVwSp+/cIR2ge0RP/SS5W0SahCfSF6aMhZB4svxBoPsl7sM5LFBqR1hvSuYkhLUY92Us8NDcGJIUfdq5JgBMs7EaY0hnSPoK9N/W5YVvyBWi/BhhofO6YgNNTpD9MfjLJmjpoyZbeYOX9JA4/u6q5o8nogFMVbla6cWjHLzbrwi+qqfc1blVcQ9qUS7qP1EEA6NBTxayc7vWoISs9V5UFKqTwCl/IIpISjbVvhN9co9dhFrztxDEGgG2SS7X6VF3nhoPZeLLmksmWkuiGoboCFryGJiYvsO/NKhFcSwxCcDKQ+HBmjFktVIM0NDcGJER7q1U7gwfJCI225FUM92sCZxuXFH1gzC8x2Vth7Odo/nBJmmyz0cJTuEQBcuKKJ474weypYLeMbjuHN8AhWtNTwRvOzRB0NqrzW4YGIP2O62+iGwGW3YLeYiAW1k1+uRzBGQxCMJhiOJWissbOs2UUdfmb/5T2quu3a36vP64liCLRmsr0hlRd58ZAeyz8Nquoqp0Ya6gOzTX2/q7wccqzgNaaJ1zQyDMHJgD60Pjc0BKMbgrBfffkhdVU85SWkkaF0h3OgPI9gb/eQ0sap1ypWeveoE0fd4uIPNJmgbiFzUaGoXZMsQLf9uA8hYOWstCF43YomoLJlpAOhKLXODI+gTnChaQttDRepkkeHR/MIhnDZLcz2jD7sRAhBg8tOUg9/ZGo5Wcc+wD5TXnmB28T/2b6BfbgbrvmNKp90eNJ9MFONFsLslHU0uOy8oBsCkxmWXARtD1WmjDTUl1Um/pL5VFYm2yZ8EI5hCMZCPAoP3zx5Vyu58hJQemgoM0dQVauqjabaI9C9ASAx1FXWQ9u6AixqcKUrhnr2qPh/vkayXGoXUhtWs4snO0+w47ifhQ3VVNstqW3Nbgdr5rh5tIJyE4OhbI+gvv1RqkSUxy2aIqtmCPZ0DrGs2VWykFmDy4YI+9QFiDn9/NhcY84RpAyBy4rtgX/hFNM+ftx4E8zTusMdbkhEsrqhpwytmaxX1HLd2fM50BNM90AsuUTl3jq2jP84oX5lCDSeTK7BRBIOPjX+5y6CYQjGwrGX4Onb1FXAZJAKDWXkCEqZW5xMQsTPjj74ydMHtcaq1hPAEKTnBw/2HC/roW2507R690LDKGEhnbqFWHxHaHTZJr3DeMcxH2tme0Zsv3BFMy8fGWAgOH4Z5uFogkg8iTfDI2DHH+k3N/CPIdXEhsON1DyC5SXkB3TqXXYskcHssBCoXoIxVg3p3cqrtn8Ddv2JPzZ8kN8GT03voHuyJ0J4aKiTJCYamudywVL1PdysewVLLgJEZcJDoV6VKNZ4OrSAiMk5MVIWGRiGYCzoJ2Y9ZDPRZJaU6aQ8giLx5WgAkDx3LMp3HtunkpK1C6bcEMievcQx0y4bCPZ3lPy4UDTO0f5hljZpJ7BETHUUN46SKNapWwSxIOe2JCZVc2ggGOW4L5yVH9C5cEUTSQlP7B3/Z0kfUZnyCIYHoe0h9tRfwp7uoJLgdnhIDvsYCMXS72MJNLhs2GL+7EQxaDmCsXsEHgLUv/oDOPU6ji77Z470h9KzpfVjRaY+PCT9x+nBw7r59ayd48FhNaXDQ9UNKldQCVnqjMbR4WiCgQgcqz0D9j8yoSNBK2IIhBCXCyH2CCH2CSFuynO/XQjxW+3+TUKI1oz7/kvbvkcIcVkl1jPh6KJQgfLCGmMm2KPi4LnVGlA8Pqt9gQ4HLfQHoxzqCymPYPDwlM6ZDR7fyaFkM12ylvhQ6WGR/d3qtaY0hvoPqHb8Uj0CLUdypnuQfd2BSRtmr4ehVufxCNbN8dDgslUkT6CPqEyVj+7+CyRjBJdeRTimDalxeFNX2OV6BFUJPzJ31oN17OWjPUMRTjcr3SPWvYOlLTUkMzWHUn0PU+8RhPqO0ZGs5ZR5XmwWE6fOq00njEGFh9o3j7+MVM8RkPaYBmedD4NH1Od9ghi3IRBCmIE7gdcDq4BrhBCrcnZ7HzAgpVwC3AZ8VXvsKuCdwGrgcuC72vOd2OgeQWCSPAK9mcyU8e8qJVmseQv9cZUQfOnwgDIE8fDkGbE8xLv2sE/OIWD2YhkuvWpoxFSyHq0EtWSPQBmCFfY+4kmZrgWfYLZrYah8HoHJJNi4vIkn9vYQT4wv2agrj3r08tHtfwDvAhqXnwNoUhN2N+Z4CAvxkiqGdBpcdjwEiNtyjNk4QkM9QxHOsR9QU+Vmn5paj14ifCKFhuKDx+iWtZw6zwvAGQvr2Hncz1BYqz5bcA4goXMcFT6JuPLiNEPQreUgoq0b1f0HHhv7c4+CGG8NsxDiHOBmKeVl2u3/ApBSfiVjnwe1fZ4TQliATqARuClz38z9ih1zw4YNcvPmzWWv9R1/fgcHfQfLftwIElEVljCZwTJ61cW4iYfVFby1Kr1NSnUlZraqcrN8JBMQDxPBSgITFpMJm0kbNm5xqPVPNlIiYyHiWDAJMMkEWKtKSlpG40niySRVVrPaPxFT/wurs3hXccaxiYVImqyEkybsFvOo8gqVIBJPkExCVYFRg/Fkkmg8Oe716M/jsJrVFZ72+ZAmK8OxBFazCY/JxG8O7uMa+WMe/X9vHvV9//QTn+bJ9ieJJ5OY42EwmREW1dx0/pzz+WbcDc9+B/6nuEHvDHZyzV+uIZghUBeJJbDIKGYBWKuQUqbWaTWbQCa5rq+bj190O6x5y6iv/5XuV/joIx8llqx8abCMhkhgwmxVk8ISSUkknkj/z2QSYsPqu5iZTC/rIPp3Wj1HPCmJxhM4LGZMibD6jFscPPXOp7Cbx9ZgJoR4SUq5IXe7Jd/OZTIHOJpxux04q9A+Usq4EMIH1Gvbn8957Jx8BxFC3AjcCDB//vwxLfT1ra+nLzxyTGA4liAaT+KuKvEf2PYPGNitWsxXv2NMaymLV3+n5hQvz/kyPPddmLVM1YfnY+AQ7HyA+xLnEXOqUsW3r3bByz+HpecqRdJJRob6Ea/8gp2us2iyhqkf2MrQqdfido5e9fP37Z0MRWK8fe08tWHvgxA6BmvfWfoCXrqLqLOBu/qWc+7yxpI6a8fLb188itdp5bLlLXnvj8YT3P3cYVbN9XDWwvq8+5TCzg4/T7f18k9nzad6YJea0XDKNeBq5LcvHsXlDNGTeI79NitrXbIk4/ti54sscC9gnmMdLTt/wnDDGlzLN/J8x/O81PUStFwJyZiqpCtSuXXAd4De4V6uWHgFTdpn8d6Xj3Jl+M9Ym5crJU/gd5uP4qlS79WDB/7Ky3ZfyR7Bzr6d+KN+rllxzZhPlHlJxuG577LbvpoVa5W+UCyR5K5nD7F6roczF9arwoznvwtzlsOC88Z2nFCfUohddhk0LmfHcT/P7NP+n0efUM2TZ1yHSVQ+tVsJQzApSCl/CPwQlEcwlue4Yc0Nebd/+vdb2Xyon8f/vYCMcS67n4OBQYg7YcOnxrKU8nj2FzBr1chjPfY9sC8ovIZt98AzP+ORxIVcuu41fOuRNt5/7WvwPPJtcC6ZnLXncPTZ3zFv4Ds8suafWR7fw9wDT/BI1ZVctGHtqI+979FHOX2ul09tOE1teOl+cC4t73XsfJLksJ8fdF/BKacu5yMblpT0sB8+uZ9zFjWwdu7IOH8xgpE43/nDg1x70TI+sWFpwf22bXuevvYon3r7a8p6/kzufGwfj3Tv4T/OvBzHL98EpkZ47VdACNp2vcSO3jaoew6fycTKutG/QlJK/BE/b17yZq5oehfLnvk6u2dvYMWGT3H7S7dz9867kVYnApQUdRFD4I+qPMkH1n6AJbXqPX/xwbu5KfZTuOBmOFXN6N23+yV2dw7xqQ0bOeI7yFFfe8nJYv0Y/3HGf2AxVe7UFunej/2vX+avi17HFRmftec2P0OyT/Cpt5+rNjz/K0i6x/69OvQMPHonLLsGFr+O2x7ay2M9bfzX2a/H6l0Ov78eWl6j9J0qTCVMyzFgXsbtudq2vPtooSEP0FfiYyec7cd8HB8Ml97qn1k1VEkt8kIE+7JLR3VG6+rUvkC1dQ2c0VqHlLClIww1s8dUORSKxnnd1x/n79tLr/TJpWv/qwCsXLuBplnqX9/ePnqD21A4pjSG9EqXZFL1I4zWUZxL7UJMA4fwOq10+ErLEUTiCb78191846E95R0L1bgmJayZMzI/kMmFK5rY0zVE+8DYdXsGQ1EcVhOOcA8celqFU7Sr/uUtNRzrU3/7TCaWukdPlIfiIeIyjsfuocGi1tUvVZGC2+4mnowzbNFOSqM0lfm1z6JbSwAnk5IFwzvVnRlT5ZY213C4L0g4lsBtr8VnMpfsEfgjfqqt1RU1AgAHD6mEdtOchVnbz2ytZetRX7rooHYh9I8j9JwpL4GaVVzntKkw2cLXqFzK/onJE1TCELwILBVCLBRC2FDJ3wdy9nkAuF77+23Ao1KddR8A3qlVFS0ElgIvVGBNJRNPJDnQEySaSDIUiZf2IP0floxBeHDC1gZoOkO+7GYyndGkqLVkcXNDI6fM82IS8LKeMB6DIWjrCnCwN8hX/75HlSKOgWjXbrpFPbObGrG5Vaikp7N91MdtPjSAlLChVatj97ereOpoGkO51C2E4X4W1STo9JU2l6DTpxqanm7rLbvef/sxPVFc3JN4zTJl6DcdGHvViRKcs2lzBySsTocSV7TUkIyrHJPPbKLVNboh8Gmy1R67Bw/qc9YXVx3dHi1p7DdreY9RSkj1q3W3TRkC33CMdbQRsbigPu0pLW1ykZRwoCeIx+5hyGQq3RBE/al1VZLjR1S1zsKF2d3rZy6sJ5pI8mq7tr7a1tTQnzGREpxT4cGsEZXOOqV0OkH9BOM2BFLKOPBR4EFgF/A7KeUOIcQXhBBXabv9H1AvhNgH/BvpJPEO4HfATuDvwEeklJM6uPRQX4ioVq2Rmss6GqE+cDWrv8uUSCibnA9HFqPMLU6EfMSliTlN9bjsFla0uHn5yEC6hLRM9LK+g71B/vxqeY1gAImkpCZwAF+1dmWleTlDvaM/1/MH+7CaBafN1wyB3pRWrkdQtwiAtVX9dPlL61g9Pqj2iycl/9hZnlrqjuN+6qttNLuLx6xne9VJun8cjWUDoRieKqvKYdUtzsoBqVJRM6akHZ/JxGz76McZjAwCyhCYNaPQFVPr1K/sfWgXBKNUDvkiPuxmOw6tuKInEOFU0z58dadkVcPplUNt3UO47W6GTYLo8OCoa9WPoa+rkgx2HQGgflZr1vYNC9Rn8YWD2ne0bqEyWmOVg9C/61WqoaxnKJKtOrr4Qji2WVUWVZiKZB2klH+VUi6TUi6WUn5J2/Y/UsoHtL/DUsq3SymXSCnPlFIeyHjsl7THLZdS/q0S6ymHzPF4fYESrhBjYfWh12WPJ7oMMyUvoU6aoWicT/zmFQ72BkedWzzk7ydAFQsbVbnl6QtqeeXIIEnvfPAfL7t1v72jg4ftn+Y9tdv59qP7SJbpFew4NkirPI5ZL/d0qddkCfeN+t5vOtDPKXO96cobXWyu1B4CHa2XYJm1hw5fqYZAhZBcdgt/frW8sNiO435Wz/GMmpittpmxmESqKWws+IajNFahwkJLLsq6b0F9NQ6riWS8Cp/JjDM5eslnyiOweVInt46oI70N8AvtMzBKL0Hu1Xp/fz/LxRGizadl7dfa4MRsErR1BdLHCJd2Yp0ojyA60E5U2EY009VW21jW7OKFQ9r6dFHHsYaHQn3qO23VjGXu0Pr174J335vuIaogM76zONMQ9JbiEQxrrnujZggmurtYb17TQkNP7u3l/i3H+eu2jlHnFof8/QxJJ4s0Q3DaAi+BSJwOUwsglY5/GdQf+itLxHE+Zfsj+7qH+Nv28q6Ot+7aTY0YpnHROrXB4SVpstIgfOwsIgIXiMTZdszHWYvSrff07lFyB/lCZsXQvqytpi76ghGi8dFzPHou4R1nzOPZ/aMbLZ1IPMHerqG8/QO5CCHwOq2pUZNjYSAU43SxR52UF2cbArNJsLSphkTCyaDZVFIC1hdNh4Z0Q9AeViemtEegOfCj5Ahyr9bjx17BLCSm+Wdk7We3mFlQ70x5BJnrGHW9E+AR9AUiVEV6GHY05S1RPnNhHS8fHlA9IClRx0NjO1ioLyUvIaWkN5BjCOoWqXGsYy1PLYJhCLqGqNauMktyy3X3LeURTHBoKCUvoU54uhTBtnZNAKzIFzAS9CmPoF5dQZw+X33Itoe08EqZKqTr+/9KHDMe327eWnuAbz/aVpZXcGyfmu1aM0frNxQCnA00UHxYzEuHB0gkZXZpZY+mMVSiaFoKuwtczbQkOpESuodG9wqODYapr7bx1tPmkkhK/r4jwwAOHoHB/Aa1rStAPCnzagzlw1NlxTcOQzAYinFq7GVVVZKnpHhFSw0y4WTQbC0p7q4neL12byoXdmRYnZhSV+uyNEPgj/pT+QEAW8dLALgWnz1i32VNNdkeQbQ0me7cY1SCre2DtIh+ROaIygzOaK0jEImzq2MoQ+Z9HB6B9j0fisSJxJMTPqtYZ8Ybgj2dQ5yxUJ0gS7rS00/M9YvVF27SQkMNSCl5UjcEx3yjhoaSwz6GTdV4NMmBeXVVNLjsPNendeaW8YFN9O5ndWI3z855L1Q38u81D7G7c4iHd5X2+qPxJPHO3epGRlzfVNPEHFuwqPbP8wf6sJgEpy/IEDzr3VN6R3EutQupi6oEdSl5gg7fMLO9VaycVcOixmr+vDUjPHTfh9PTtHJIJ4pLOzl5nbYxh4aklAyGoqwIvgjzz05rUWWwXDMEPnNplTh6jsBtd8PwAEnMHAmY0tsAX1L7zpTpEXj6t3JQtlBT2zRi36XNLg71BXFY1AWMLzZ6GEtKOSEewZajPpoZwFk/N+/9Z2rnjhcO9av3vLpxfKGhjEQxTPysYp0ZbQgi8QSH+kKsme2hxmGhrxyPoLoRXE2TExrSdIb29wQ5NjhMa72TY4PDDJuqis4tFhE/SV2CGhV+OH2Bl8ePqw7FclzYoU0/IykFvhXvhDM+QEv3k7ymto87Hm0rqez21fZB5iXbiVlc6UQ7QHUjsy1DRdVANx3oY+1cT1rCOdin/g/l5gd06hbiCqqr+FLyBMcHh5nlcSCE4Mp1s9l0sC/tSfiOQvfOvI/bcdyPy25hfp2zpGXVOq0MjtEjCEYT1CYHaA61pZqzcjlrYT0mWY3fJEoyBL6IjypLlWrOGh4kYq0hGE0yHE3gtDixCAv+hPadKSdHICWz/NvYbV6RN3eytFlpDvkC6gLGHx/9fxROhIklYxXPEWw5MkCLaQCLN2+fK7M8VcytreJFfWJZ7cJxhoayDcFEj6jUmdGG4ECPUmRc1lJDg8ueEnkqii4q5axXxmDCPYIedSyTKRUW+tBGVcbWOWwpOrfYFg9gcmR/MU5fUMuh/jBx9/zSK4eSSew77+Hp5BpmzVsEZ7wPLA4+1/AE24/5eXzP6Mbwuf19LDEdRzTmhHOqm6jDx4HeIKHoyPLdUDTOq+0+zl6UERbSx1yWWzGkU7sQS7ATO9FUaWgxOgbDqaqeK9fNIilVlzOgQoOhvrxiYzuO+1g1242pRNkIT5VtzIZgIBjlfH2SVQFDsHauhxvPX4NfQLJEQ5AKtQwPELN5ASWGJoTAbXfjS2ifvRKqhlJX676juBP9HHbmSpIp9DGkXYPq9OSTUaXDU4TcPoVKIKXkwNHjOIhCTf6ucFBewYuH+jV139ZxGIJ+wyOYCtLj+lzUV9tKzBH0AkJVELiaJiFH0JeRKO5hUWM1l61WH8ojQe3flydh7A/HqJIh7NXerO16eKXfXkZT2ZHnqAq284fEBSxudKn1nPJOFh3/E6s9Ub71yOhewbP7+1hu7sDSlHPydjXiiqsv0e7OkbHglw4PEE9KzlqYkSjWxebK7SHQqVuIQLLI2jeqIfCHYwxF4sz2qkqOZc01LGt2qfBQJJA2wn37sx6XSEp2dZSWKNbxOq0p4bhy8Q3HeI35VaL2OmhZV3C/OoeXpIBAiYbAa/eqG8MDSK1qRr9gctvc+GMh1ehUpI8glogxHB9OX623vwhAtzt/N/mixmpMAo72qkS+3zR6cltPKFcyR3CoL4Qjon2/C+QIAM5sraMvGGV/T1CVkPraleRGOejViM506SgYhmBS2Ns1hMUkWNTgoq7aVlofQahPjeozWyYvNFTdQDiW4PkDfbx2WSNep415dVUc9Gv/vjxXYwe7A9QQosqdPUhk9WwPNrOJI4kGlSwupZt666+ImKp40XEutdWajMDZH0EkInxl3ia2HB3k6X2FRcfCsQR7jhyjQfaPjOtXN2JKxnATypsn2HSgH7NJsKE1s2JorxKa88wbsX9JeJVW1Vqnj85RcgQdWg/BLE9a8O/KdbN58XB/diNc376sxx3sDTAcS5ScKAbwVlkJRhMlVTLlMhAMc4FpG0NzLshWqc3BY1fr8UVLqxrS9yc8iNAkqPXvidvuVidga/EO99RJWr9ab99MGBuR+pV597dbzLTWV7O/e5gas6MkQ6B7BKn1VoAtRwdoEVppaBFDoOcYXzzUr1UOSVVEUA45/UK9gQgWk8Bbqv7ZOJnhhiBAa0M1NouJepedvmApoaF0Zp9qzSOYSJmJYC84G9h0sJ9IPJnqQF0z28PeAe24eQzB4e5+bCKB21OXtd1hNbNmjpvtw3XqyzVa80s0BDvu51n7BcxuzAjPNC6DpZex9vjvaXULvv3IvoJP8fKRAeYntJNm7lV8tUoWtjpCeSuHnj/Qx5o5HlwZIx7p2QP1S4qe8IqiGYJljoFRk8V6D4EeGgJ4w7pZSAmbtmXkBvrash63XR9WP4q0RNayNCPrGy4/PJTo2EaD8BNrLa6XpZ8o/fHS+ghSJ9bhASyubJ18j82jOoZt1UprqAB6V7HuEcj2F3k1uYh6d+F6+KXNLvZ2D+G2VOEzj95dPBEewZYjg8yzaMd1FzYEixqqaXDZVJ5grJVDufISWjNZqWHF8TLDDcFQSn2ywaVCQ6OWQwZ7012+riaQiYkdLB1UHsGTe3uwWUycrZVQrpnj4VCgcGjoeKeKYXvqRtbZn76glk2D2hdmtPDQ7r9AdIjfRM5VYaFMzv0oItTLl5fs5oVD/Ty7P79X8Nz+Ppaate7h3ASv9sE/rT42opdgOJpga/sgZy/MNmb07h17fgDA1QJmG63m3lGTxcd9uiFIy40vbnSxcpabHXs14ydMWR6BPxzjZ88dwmkzj3zPiqBf/fnGUDnkbn8CAOvSi4rup5/YB+PDo17A5BoCm0urrgtmeAQRnzaToIghyIzfxyPQ8SovJ5fQ6CosUre0qYbDfSHcFpfyCEYZYj8hHkG7j3UeLeRVxCMQQnBGax2bDvanZl6UnSfIlZcIRGioKWEOd4WYsYZgOJrgSH8oNeSkvtpGUsLgaFdjmcOlXVrpW3CC8gTxqNIZcjbwxN4ezlpYl+qsXTvHQ1BqJ6c8HkF3rzopW6tGfjFOm1/LwbhmIEb7wG79NUn3PP4RWsKixpwruNYLoGUtZ3f/mlk1Nt5/92bufGwfkXi2Ssiz+/s4u6ZPldvqV0w62nu4xhtld4c/azjLy0cGiCVkdqI4ElCVOmOtGALlSXjmMZseuv2Rosb/+OAwZpOgqSZ77sSV62Yx1KcZt1mnpHIE/cEo1/5oE6+2+/ja29YpwbAS0UdMjqWprKn7GXYl51PTmL/MUScVGjIJKFKfr5djemweNdci7MdSXUeNw5KKX2d5BEVyBFkeQed2RCLCluSSovHvpc0uEkmJzeTCV4LeUK6W0XiJxBPsOu5nWVVA5QMzZ4Hk4YzWOo4NDtORcIOlqvwS0nw6Q5NUMQQz2BDs6w4gJSmPoF5700ftJQj1QbV2YtLCGhNWOaR9OAZMHvZ1B3jtsrQC6Zo5HgJoH848ekN9fdrVeZ4qitMW1HJUas9VzBD4O+DAY3QtvBqJaeTVrRBwzscw9e7lgdeHOX9JA//74B4uve1JHt7ZhZSSYCTO1qODrHV0qd4Lc44ypPYeLnGGiMSTSjpDY9OBPkwiQ2gO0iGYsfYQ6Hjn05joIppI0l8kQdsxGKbF7RgxMObKdbNoFD4kAuadDX376fKFeMcPnmNP1xA/fM/pXLludnlL0iaLlV05FAkwy7+V58R6bJbiX2k9PDPayTVTeVTtJ6GqlgaXPcsjCEQDJKzFp5TpUhVuuzuVKH5lNEOgqcwKqXsEo4SGIj7MwozLWroHVoydx/1EE0nmWAaLegM6S7RKp6MD4bFVDhUTnJsEZqwh2KNXDGlzW+u1+GxRmQkps2p908JzE5Qw1hLR2wfUleJrMgxBXbWNGrdX3chxy6WUDA1qHyzHSEPQ7HZQW1vHkNlbvIR02+9AJtladznASI8AYPWboWYWjdt+xA/fs4GfvfdMLCbB+3+2mRt++iK/23yUeFIyN340f5WPsx4QzLWpE0lmnuD5g/2smeOhxpGRMBurxlAu3vm4w+qKvljl0HHfcFZYSGdBfTXLqofxCbcKU8WH+cj3/szxwWHu+uczuHBFc55nG2VJmkdQduXQoacxyzhbbKePumuqEcxcPNySqTyaUth1eGlw2ejN8AgkkoC1qmgfQZZH0P4iIUczXdTR6Co83U+vHIrGq/GXIInhj/qpsdWUNGynFLYeHQSgLtlXtHRUp047f/QHIyo8NKYcgYCqWpJJSV8wahiCyaCtawib2cQCrdkn5REUSxhH/Ep6OmUItBPzhHkE6qp+U5dglseRqq/WaZ2leSQ5V2Nd/gi2hGYcCtRVn76glsPJRmShKxcpYcuvYe6ZbB1uwGoWzMvXGGWxwVn/AgefgM5tvGZZI3//19fw2Tes5OXDA3z+TztxmhNUBY7kNwRmCzjrqMWHzWJK5QnCsQRbjgxml42C6iEQ5pSK6Jjxzsce6cNBpLghGAxnVQxlsqJmmM6Em1eGVZitPnyYX7z/LM5dXKb+kb4kp54jKNMj2P8IEWHnaM3ow32sJivVZseoHkGWIdBzYFW11Fen+210o+K32oqGhvTnqrHVQPuLHK9ZA1A0Bu6wmllQX40/7MRvMiFHUdz0R/wVrhgapKnGjjXYBe7RPbt6Ld/RF4ymPYJyxgCH+pR2lsnMQChKIimN0NBksKdriMVNLixaDDf1jyzmEeRKQju8ar7oROUIgup4Tx6D1y5rHHG1s2SuuuqMBLO/0Ad6AtQI7YuZ0VmcyekLajkQbyDReyD/B7ZjK/TsglPeyYGeAPPrnIXj3affoEoIH78VpMRqNvH+Cxbx6Kc3cu1Z8/nkaRaETBRO8FY3YQ72sKKlJtVh/MqRQaKJZHZ+AFTFUN2iotOwSsK7AIA5ordgCWkyKen0hbMqhjKZbRmiR3q48S9qzZ8/z86p82vz7lsKLrsFs0kwUK5HsP9RtlvXUu0sLSzitbpGHfiSkqC2ZRuChhpbKjSUCjNZbcWTxVE/NdYazKF+GDzMAfsqqm1mnLbiA2QWN7rwhWzEhSCkiz0WOUYlK4b2dgVYPasaEegqzyMIRFUJaSxUXo9RZlexZmgbDI9g4mnrCrCsOf3FqXXaEILiMhOprmLtik8IrYR0gkJDmkdwJOLMCgvprJ5Xz7C00def/SU50BukBm36Vp7QEKiE8QE5C4v/CNx5Jjz2Zejend5h62+UkVv9Zvb3BItXv1TVwmv/HXb/GXbcm9rcWGPnS29eywdWal2hDQVGNVY3QLCX1bPd7DzuR0rJ8wf6EILs/gEYf8WQjlZCusDUW7CEtDcYIZpI5g0NAdjDPeBqxOpuIWl10RIbfcBOMYRQdeNl5QgGDkPfPp5jfcqjGA233a1CQ0XCLdnKo4NqY5WX+mo7A6Eo8UQy7RGYLEXLR1Ndxcc2A7BdLCsp7LG4qZp+vyqO8IeLG4KsLuhxIqXkaH+IVe6oqgosIUdgt5hx2S1pjwDKCw/l0xkyPIKJZSgc49jgcGoIBiiZ3lqnrXiyOKUEmnGV6ppAmYlgD0nMBEwuzlsyMtywZraHAA58vhxD0BOk1qwZggKhoRUtNfzc9Cb+PPfTKtfxxNfgu2fBnWfD41+Fbb+H5a8nbvdyuC/I4qZRrjbP+RjMPg3++umRhlGXhKgvYAhcTRDsZtUsNwOhGB2+MJsO9rFqllsNWtFJxKD/wNg7ijOpVR7BCsdAwRJSvZlsdr7QkJQQ6OHstat49N9fh6lhyYimsrHgcVpHr1zLZP8jADwaW12yIfDYa0cNDWUpj2Z5BHakhP5QND1tzGwa1SNw27REscnCK/H5pRmCRhdxfaraKDMJ/FF/xeQlBkOqm3xJlVaEUYIhAOUVDISiYyshnUJ5CZihhqCtW8XUlzdnh03qR+suToWGMq5Sq5smMDTUi1/UcMq8uuwTokZjjZ2wcBIcyv5CH+wNMKcqptRJTea8T20xm1g+r4XvD2+EG/4Mn9oDr/9fdXX/+FeUN3LKuzg6MEwsIVnUMMowDLMFrv6uqmD666ez7+ttA/fcvIqYgKbZ1MMqrQv3lSODvHJkcGRYqP8AJOOV8Qiqm8BsZ7Gt8KQyvZlsVj6PIBqA+DBWTzMOq1l5OzlNZWOh1mkrT4p6/6NI91y2hJvUmMoS8FbVl5wjUMqjg2qjw5uq/e8diqab00wC4mFVZlrguVIVQy1rOR4o7SS3uNGFTChD4I+NLjFRKY/gSL8Kq7batGMWaSbLpE6XqfHOB0R5JaTB3tR5Rc/BGIZggtnbqWsM5RgCl614sjin+w+YUL2hqL+bzkRNVtloLtJWTSSU/SU50BukxR4tmB/Q2bCgll0dQ6pKpaYZzroR3vs3+LedcO09sOwy9mtGc1SPANSMhtf+J+y8LytERM8oktHVjRAdYkW9BSHg1y8cIRJPjkwUj1djKBOTCbzzWGAq3FR23FfEI9D/53oJcf0SNZegzKlvuXirrKXnCBJxOPAksdaNJKXIe7GQD4+jdlQp6sHIYFp5NDyockAWW1ZRRcojSI2rzO8V+KN+3NYaOPYyzD2j5Br5xY3VaUNQTG5dJhmKDlUsWXxYMwSzTINqQ4keQepC0mIH95zSQ0M51Yg9QxEcVlN2N/0EMzMNQVeAKquZubXZX/D6jBrpvIT6VNzclnFSdDUpaz4BMhND/Z30y5q8+QEds6MGEQ0QiKg4fDSe5Gh/iHprpGBYSOeilc0kkpJHduUYMvdsWHoJCMGBXs0QNJRYn33ev8Ks9fCXT6v3RUrlERQ7eWtNZdXxARY2VPP0vl6ESGu9p+itoCEA8M5nluyiq5AhGBymymrOH3LRDYErwxAgldcyDjzlSFEf2wwRH77ZFwCU7BG4bW58JkGySCVOrvIoVSoJrssi9wYi2Mw2qixV+NE++wVKSH0RHx4JRAPEZp2GPxwvSV7Z67Th1Rvg4oWrkgKxAEmZrJhHcFQzBPXJPtU1rhv7UajLFK6sK0OOOjKkqhFz5CUqVQpbCjPLEBx8Eo48z96uIZY2u0boeDSMGhrS5CUy/0GuZk1mongyaywkAj0MmT2snVP4Ssde7aaaMLu0sssj/SGSErymcMFEsc66uR5meRzZE7dy2N8dpMFlSw23GRU9RBT2wV//HfzHVCKx2Mm7Wi/D7WHVLLXmFS1uvLkntp69xUNM5eKdT12sk6FIPGVIM+nwDTPL68j/hdTzQlmGgHHnCbxVttLLR/c/CsJEZ8M5ANRWl5oj8JAUgkCRBKwvmqk8OqiEFklX1/UOqe9Jja0GH9p7l8cjkFIqjyChwkYDDpWkLzXssahOfTaKzSSotLzEkb4QDS471lCnFkIs7cpcNwRKjnpB6aGhnGrEweFYyUa9UswsQ/DXf4enb2dP19CIsBBAXbUd33CssPpjpryETuokVtmEcTIpcUQHcHhbRnS1ZuKq8eJiWI2uRJWOArgIjeoRCCG4bHULT+7tyTsLAOBAb4BFpXoDOs2rVYhoxx/hya+rbUUNgS7V0cNqLU8wIiwE45tKlg/vfKpiA1QRzttLcGwwzJwCpaMp1dnqXEMwvjxBrdNKIBInlijBwzz8LMw6hT4tfOKpKjFHoJ3gfVqJaD5ydYZ0j6DGbsFmMdGrhVA9dg9+WdgQDMeHiSfjeLT8QY9Uz1mqIVjSWIdJCnzJaMG6/ErLSxzuD7Kg3glDnSWVjurUVduIJpLqoqJ2ocodjjK5DciecQIEI3Gq7flzexPFzDIETatIdO2kZygyIlEM6audgjHazK5iHf2KsMJ5gl3tvbgJ0tBcvJnFUe3BZYqwXau/1yUaHInAqB4BwKWrm4nEkzxRYLjM/p4gi5tGSRTn4/x/Vbr4L/1U3S6W4NVzLsFu1s1VJ4pzF+e8z8mkFmKqQKJYJ6OXIF/CuEObTJaXQLcWNtDWbnepWHLOXIIRxMKw476CJ7WSm8qkhK7t0LIuFUqqLblqaPRZwCMNgRdQFw8N1baUR+C2ufElCo+rTJ2kY2r/zoT6TJZsCJpqsCYs+E35n19fa+brGi9H+4fVVDl/R0nNZDrp7uLMEtJDoz8wlF2NGIwkqB6lx6LSzCxD0LwKs+8w1QynxOYyaRitqSxTeVQnJTNRWUOw77CSfmiZVVxEDJuLGhFOzcc90KNCOeZoYNRkMaihGrVOKw/mCQ8NBKP0B6PlewQAZitc/T0wWcDhSXtO+XClPYJzF9fzy/efxSWrciQaBg6qGHRz/qlWY0IzBHNFz4iEcTSepCcQKdhMRqBLmxyXceVWv0QZq2K8/DP4/fXQsSXv3R6nrjc0SsJ4qEOdoJvXpPYdEUorQEp4LlaiIQgPquZJjYaatGS7x+bBn9Deuzy9BKmTdCwMDg9dWqi/1BGMixqrsSStqsqpQN9DJT2CaDzJcd+w6qIf6ijLI8jqLtZLSEsJD+VUIwaj8fRY1kliZhmCptUALBPtLG/J5xGMIjORzyPQT3AVLiEN9qtQU03dKB9EuwuHDLOve4hQNM7B3iALG6rVl6aEumqL2cTFK5t5ZFf3iJBYKlE8Fo8AoGUNvOGbqsegWOLLWgW2Ggj0IITgvCUNI+PyHVvV71mnjG0t+dCayubm8Qi6/GGkLFAxBCo05MoxVvUl9BK0Pah+FzAYuhT1qAnjrh3qd/PqlFppyVVDNl2KOn8CVkqphtLYRoaGgKyxrm67G19c61kp5hGEA+BqTj2uvogEdSaLG12YEnalN1SgyilV6loBQ3BscBgpYaHHrPJ+NeV4BOr8keouhhI9At0QKO9ShYYMQzBxaFeTp9iO0eIe6fLrrl1ejyARV1dGmaWjoK52zfaKewQRvzIENvco4mU2FyaS2GWEXR1DHOgNsKTeoa6eHaW5ypevaWEoEh8xT2B/t/pil6OpP4LTr1ddx6NR3VDcmHZsVTLWjfmnWo0JVxNYHCy19tKhzR3QOZZnIE0Wge6RXk7DUnXyyDO/GFB6PAefUn8XMBhp4bnRDMF29bt5FYOhKG6HpWguKZOURxAP5w1RheIhFde3eyA2rHoEtNAQZPfbeGwehuLB9OvLIZXIHfaBq5meoQhepxW7pbQY+NxaJ6ZkVVFJjJSoXcc2+NqicX0XD/ep17KoSitXLccj0ENDoagynHZPaSWkIU2iXfPgA5E4LiNHMIF45jMsqjjD2Zm3EqShOl0aNwK9uzLXIxBiQnoJkpohKBpSgVQFjYswz+3vpTcQZVmt9uUusdPyvCUNVNvMPLgjO+G9vzeAzWxibm0esblKM9rYz85XVZ/CeDWGMhECPPNYZO2n05f9P9cNQ95mMlD/bz2kpTNa5dDBJyERAURBj0CvFhm1u7hrh6qgqqpVVSbVpb8vKQVSE+pEn0O24Nyg2pjpEdTY6Quo6hi33c1wIkIU8kpRpyaHBftThqAc6QSzSeAw1xQdTuOP+LGZbDg6t6uT6sEnS37+XPTS0VImk+WSlSMQAupaS/cItGrERFISjiVH1WGqNDPKEEghaJNzWWE6mvd+d5UFi0nkH2KfSujkqWapbqx4aKg6eIQkplT4oiA2dRUxpzrBA1uVrPJitxbiKSFHAErpceOKJh7a2UkiY0jL/u4grQ3Okq80x4XWXZwXKZVHMKvwUPYx453PHNFDpz/7hHh8NHmJYBFDUChP0PYP1ZjVen5BY+EpVYq6a4eqzkINsilntq3VZKXaZCvYXVxIeVSnXquO8Q/HU+Ejv9mUt48g5REEVCitJ1C+zr7L5i06k8Af1ZRHh7Q816Gny3r+TI70h7BbTHjj2ve9xGYyAKfNjN1iSp8/altLyxEE0yHnoFa9N5nNZDDDDEFPIML2+FzmRA/mdYmFEKq7OF9oKFd5NBNXc8WF5xrDh+i3zgJrYc12QE2HAtY2Wtjbpa7IWl1aq38JVUM6l69uoTcQ5eUjaU2XMZWOjpXqxsIegf+4ev9nra/8cb3zaUp0jfAIjg8OU+u0pibCZRHxq3BJbqORd4Fy8fOd5KWEtodg0UZoXqOqi/J8Bms0BdKioaF4VInvaYbAF4qWnChOLdXiLBhuSQnO2bJnEejoJ/LeYCRDeC6/3pAvqgbGOCMBcDWlmqXKWquzgSGziUiBkFuq+W1Imxh3+Nmynj+Tw30h5tc5MQU0o1KGIRBCZMvU1C5UQ+wLSG+kCPWlE8VaP4uRI5hA2roC7JHzccQGC9b911cXGGKfk9DJosLCc1JK5iTaGahuHX1nLTS0ul5dtZsESl4CSg4NAWxc3ojNbOLv29UXIJZIcqQvNPZEcbm4mtR7nMjTz6AnilsmxiNwJXyEAr6sZHlHEfnplNHPTRabLapaJF8vQc9u8B1RHdv1i1WFzVDHiN2EUFIRg8XmFvfuVZpLmR5BqQ1/Gm6rq6ACaUqCuqBHoBmCoUhairrA3GJ/xI/H6kJAKllcrkfQ7FVx+uN9I98vyOMR9O4Z84XZkX5lCBg6rnJ/Ga+7FGqrbWo4DajPQjKmmiqLkVGEEowoo2H0EUwgezqH2Cu1cky96iKHepct/5SyfMqjOtVNKnQ0muUvkcFAmIV0MOxZPPrOWmhoqVcZgnl1TmwxLVZbhkdQ47By/tIGHtzRiZSSw30h4kk5uR4BMm1wM+l8FRCqCqnS1KZ7CbqH0pVDxweHCw6kSYUBXXnyN/VL8vcStP1D/V56yai5BO9oMhOpiiH1fgyGomV3onps7oKhofzKo97U/fpAmb5gNGM4jaOgR+A2K682bG8gFE2UbQjma5VzxwfzX2yl1E39HVCnfWcOP1PWMSAtP61KR7VmsjJlHrJkJvRegtHCQ6G+VBFKyiMwcgQTR1v3EF0O7YPSvTPvPvXVBYTnUt1/eXIErmaQyfwnMZ1gX/pLNQr9HfuwixjJugKyzZloHkFrjbqaVaWjWn14mbK8l61upn1gmB3H/akO5ZLE5ipBqgw3z5Vcx1ZVkWObAO9E6yWYJ7qzSkiPD+YfUQmkvb98GjS6Ici9KGh7SJ24PXPTcxmKlJAWbSjr2q40r+qXEE8k8YfjJZeO6ngc3lFzBFnKoznlo6CKKtIegaNgjsBt0gyH8ALl6+wvrlfvc18w//crpW461AHLXw9W55gMQX8wSjCaUF3FvnYlHFcm6vyRERqC4gnjZEKdFzK6isEIDU0oezqHaGyerU7cXQUMgcuu6oBzCfWpE6slz4c4NbKySML4N9fAH28saZ3Dx3YBYG0uoYtWE8Crs8ZorXdy6rzatLtfpiG4eGUzJgH/2NHJ/h6tjC7fnOKJINVUluc97Hi1sv0DmaR6CdJNZYFIHH84Xn5oCJQhSETUiUQn7IMjzylvAFRtuqWqYBey12krrkDatQMaV4DZgj+sThyldhWnjuGo1+YWD464zxfxpZVHhwfUaNCMz1Kt04ZJqPneKY/Akn9KmS/qw62dZrrLlJfQaXYpIzRQQBLDH/XjFjYVhvHMg3lnwqHyDYGuOjq/zqnCbw1Lyn6Oump72iNwz1ENlcVKSIcHAZkyBLrmlZEsnkBOnV+rOlabVkF34dBQMJpgOJpzRZeR0BmBfkIoVDkU9ikt9vYXS5pjmtAGtNfMKaGLVrtKFtEAf//X1/DRC5ekr/LKCA2BMoJnLqzj7zs6OdAToLHGjttR3glmzKQ8guxeBoJ94G+fmPyAdlxpcTBX9Kb0hjr0OQQF5SW6lLxEvs+DfrWfmSfY/5iK6S+9VN02mYo2n406paxrRyosNFBmV7GO29mAz2QiOTzSIxiMDKabs8KDqh8lI0RiNgnqqm30BiK4rC4EAp/FWjhHICUIMx0xVYZcbrJYX0swNrI8NZaMEYwFcetfK/csWHC++n4X6ucogF46urBqWH3fxyBnUu+yEYomCMcSKmfknV88NJQjLxHSzjtOI0cwcfy/K1fx/gsWqSRbz568MX29KWREeEhXHs2HHiIo5BEcfUGFjoYHVAXMKFgH2uiVbhqaS6hY0CWxIwEcVrMq9Yz4VaIrn/cyCpetbmFvV4An23pYPFneAGSI9+W8h50T0FGciRDgnc8Cc0/KEOjNZIUF57RmsnxDf1Lx/4yr/baH1Ml07pkZ+y0uKFDnLTacJtgLgc5Uolg3GOUmiz1V9UqBdLh3xH3ZyqMDeROm9dV2+gIRzCYzLpsLv9mcNzTki/pwx+NQ3UhvUF3tlusR6A1wITmslD0zGNL0knRRO2pmQet56u8jz5V1nCN9av1zElp5eeOKsh4POb0EoMJDxUJDOfIShkcwmTStUuV/ebTj9YqIESWk+eQldEYLDWXGK/WO0CK4/Ac4xJzSmkrMFhVmyBQQC/tL7iHI5bLVKjHX5Y+waDwdxeXi8Ki4d65XlaoYWjthhxbe+bSa+1JD7PUQ0axioaFCGvXVjaqjVI//J5Ow7yFYfFG2nHH9EjVvOD4yBOR1WhkqpECaIS0BlK0zpJPqLs4jn+6P+PMqj2bSUJMuqvDYPCrfkOMRJJIJAtEAnlg0VTpqEumTZanoHkFMRFKD3XVS+YyYtr2mBeacDhZH2eGhw/0hmt127APa/24MSrcjDUFr8dBQHnkJMHIEk4MuXJanckjXQBnRVBbqz186ClruwFG4hPTws2lphM5XR11e3fAhOqyjNJJlHd8FkQy3OTJUdlhIZ7a3KqUAOi5piXIRQuslyLlC7diq3OtCYblK4J3PLNIewfHBYUwCmgtduQa68lcMgXod9YvTYZ/OrWp/PSyk07BUzbHIc7VYVIF0hCEoT3k0dQztit+fJ+4+GBnMMASDWRVDOvXV2XpDfsEIQxCIBZBI3NFQqqu43mUvu0HRZrZhx0TUHOdAT/YxUvISUa0h0NWiPOG5Z8Dh8hrLUqWjPXtVwtk9iuBjHtIRhYwBNWFf4UKRnP4k3RA4rUZoaOJpXKFivHkqhzIrIrLImCk6AiG02cV5Kl6iITWib9llqkKlcxSPINiHK+mnv2pBKa9EYXNlfwlLFJwrhO4VTFqiWKe6caRX1fHqxOUHdLwLcCf9+Hzqy3p8MEyz24HFXODrkU9wLpOGpWlD0PYQIGDJxdn7FCkh9RQTnuvaod4nLbk+1hxByiOI5q8aShmCYC9Ujfzc12aUuLptbnwkRxgCvQzVHR6CmvLlJTJxCxsRc4L9Pdl5gqxjOBvSEiQLzoXObUXHceZytD/E/Lpq1YfQsFTlcsok7RFo54/RSkhHKI8mcNrMI4ZmTTTjMgRCiDohxENCiDbtd97uCyHE9do+bUKI67VtTiHEX4QQu4UQO4QQt45nLWVhrYK6RUU9gqyRldEQxIcLh4ZAayrLExo6tllVMyw4T4U3OrcVX1uvShSH3CX0EOjYXNk6L2H/mD0CgHecMY/3nLMg/3CYiSRXqiPsh/79E9NRnIlWOWQfaieZlFrpaIGwkJTqCr+YBlT9EvAdVTo+bf+AOaeN9CDqtf9vHkOgn9R9+ZrKuranvAG1TwyTUB3J5ZBSIM2ZSZClPBqPqkS9fjLLWaM/HCORlGo4DYkROYJUh/LwYEpeomGMA9m9FgdBExzsHMx/jNBAti7QgvNUXu7I8yU9fziWoNMfTnsEY5x7MSK0rJeQFhphGuxTsiNW9XmbCuVRGL9HcBPwiJRyKfCIdjsLIUQd8DngLOBM4HMZBuPrUsoVwKnAeUKI149zPaXTtCqvR+C0WXBYTfRlegT5htbn4mrObwgOPwsImH+WMgT9B4pOLZLagPZ4XRmla3ZXuncAxu0RNLjsfOFNayZd+Co1/1lHz6dMhMZQJlovQbPsoj8UVSMqC1UMhX2QiBb3CPSr/aMvQPvmkWEhUHF3Z0PehHFtIQXSZEJ1KDenG+sGNHmJcq8gPQVmAacmitk9Sh5BJtVFU541SqkMkdvmVlPKosGsqrjU1Xo8rrqKx+MRWJz4TCY6u7PDr6ljBHqz5SDmnqHkPkrsJ2gfUPLTi9xJZfyKDVIqQo2mApsKLTcsVfmvvQ/mf0BO7lEpj558huBNwN3a33cDV+fZ5zLgISllv5RyAHgIuFxKGZJSPgYgpYwCLwPlB+XGSvNq5a7lOSkrmYmMq7FiOkM6hYTnDj+jDIDDo32BZcEeBoBo127C0oqtvtzQUE6OYByGYMrQ9Yb0k8lEzCDIR8Zcgk5fmOO+EkZU5grOZaIbgk0/AGR+Q6Dvl6eXwFulT8rLMQT9B1SRQ4ZHUK7gnE5KgTQRztqeJS+hX8XmMQTejAE6HrsHfzKKRGapmaau1pMJZHXTmATndDy2GvxmE3192d+xlLqpvytbMtrmVEnjEhPGeunoEpNW1TdGQ2AyCWqdGd3FFjuseRvs+lP+MFVOWXpICw1NNuM1BM1SSl0ApBPId5k0B8iU+2zXtqUQQniBN6K8irwIIW4UQmwWQmzu6amAwFvTKkCqK6wcGnKF53JqffPialb/1MyS1HgUjr6o1CYhXflSJGEc79rDATmbZk8Z0s+5yeJxhoamjOpGdbWtf2E6XlXvaxma8GM7bgNJs4O5oocdx5XmUNEeAn2thdDDPnv+qvYrFNpqyD/RrKACaWoGQUZoaAw6Q6ApkAqLmgWcQZbyaFFDoI45EFIeQRxJSIisC6vU1XoyScBaRywhx2wI3JokRsjfr2r0M45Rba3GEuwZOUSm9Tw4/kr2d6MARzRDMCeunarGMRI1q7sYYP21KrS8476RO2fIS4DyCE7I0JAQ4mEhxPY8P2/K3E+qAt/Ru6VGPr8F+DVwh5SyQCANpJQ/lFJukFJuaGwcRaO/FPQvU56r83pXjvBcznDpvLialBudGdro2KI+AAvOVbe981VpYZESUnP/PvbLWTSV84XJTBYnk+MODU0ZGSMrAeURTHSiGEAIkp75zBU9vHx4EChWOqrrDBUJDdmqNXkCCUsuKZx0rF+ivMicK8UauwWTyFM11LVDdflmnKQGxqA8quMx2/Ehs0pYs5RH+w8oLas8IdHaHI8ANAXSjHGVqav1ZJI+vEB6HGzZa7V7GTKZcBFKzeUGTXDOUg3IkRcMC85VlVlHN436/If7QlRZzdQMHVDdwPqoyTFQV21jINMQzDlN/c+2/GrkzjmhoeCJGhqSUl4spVyT5+d+oEsIMQtA+52vkP4YMC/j9lxtm84PgTYp5e1jfhVjobZV1d/nyRPUVed6BCWGhiA7PKTros8/R/0WQhmgQpVDsWHsgaPsS86hyV2uIdCueqIBQI65j2BKyWwqi4WVtzbRYSENc90C5onelAx3wdBQyhAUCQ1BOjy0rEBYKHOfnPCQySTwOm0jcwRdO1TMOUOafHCMHgGAx+IcoUCa5REMHFQnxDzCa7oh0D0C0GYS5HgEDmHGLqFLeoHydYZ03FV1DJtMuMRQVuWQP+JPidqNGDQ/7yxlOEvIE+ilo6J3rxKuM4+9o77OZcsuPxcC1r8Ljj4/MhQY6h9hCE5Ij2AUHgCu1/6+Hrg/zz4PApcKIWq1JPGl2jaEELcAHuBfx7mO8jGZoWlFwcohfQIToK7yhSlLk30EqSH2Gcmsw8+qUtXMK6qWteqYyTzNQn37EUj2y9k01owyhyATu2YIpEwnjU/W0BAoj6B7h7qam+hEsYbwzmeeqYd92kmmYGgo2K1OLnlKKrNoWKauLBe9rvA+9bocRZ7KoSrryCllORVDoK7I9ZxCuXgsrhHCcyNCQwWujDPDV6nEs8mUNa7SF/XhFlawOumLqv3rxugRuKvUybLK7E+NUE0fQztx5noE9hqYvb6k+QRH+0PMr3eq0tExNJJlMiI0BLDuHeockukVxCOqETQjRxCMJqg+CXMEtwKXCCHagIu12wghNgghfgwgpewHvgi8qP18QUrZL4SYC/w3sAp4WQixRQjx/nGupzyaVufvJai2E00kU+3ehPrUF79YXbF+hagLkiUTqnRNDwvptKxR7nO+bkOtdPSIaS5uRxlXBTaXCkvFQmMWnDshyAwNdWh5lEnyCKhdgIcALqkmVBXsftVnFY9WY37Bv8F1f8zbjJWibqE6OeTrJXBas3MEYZ+q4skwBNF4kmA0UXYzWeoYKSnqwdS2lCGwVKvO5zz5ASA1I3kw0yMwmbKKFvwRP27UKNc+zbupG2sYq7pZe/wwB3qzj+FB81jyDZFZcB4ceynvSE4dKSVH+kMs9FqU8RuDtEQmddU2fMOx7M5w9yzVXb711+k8Yp5Iw0npEUgp+6SUF0kpl2ohpH5t+2Yp5fsz9vuJlHKJ9vNTbVu7lFJIKVdKKddrPz8e38spk+ZV6qSTM8Qi1Uugh4dyEjp5yQ0NdW5T1n7Bedn7pRLGefoJeveSRBCsWZB3pnJB7Gm9odRc15PRI6iqA4RmCLaqSitvGdVT40GrHJojepntrSr8/ge6C3cVZ+KeDYteW3wfi10dt5BHkBka6laKtJmlo/rwmjGHhhxeTYE02yOoslRhD/ao/pcChkAIgbfKykCuRxDL9gg8ySS4mlMx87HmM9zORu3xw9mhoagfdyKpvK98nf+t56sChPYXCz53TyDCcCzBanuPuqAaR6IY0t3FIxRk179LDanRZyrnyEskk5JQNHHyGYKTniZNaiJHibRei2OmEsbFdIZ07DUq56DHkHV3VM8P6DSuVKGFAoagx9yM1+0p51WkhtMQDZzcHoHZot7nQHc6UVzmYJAxkyFHXXAOAWizioskisulPn/lUK3Tlj2lrEDFEIz95Opx1GkKpIOpbSnl0SIVQzr6AJ2COYKoX+shaKI/GKXGbsFmGdspx+NQrUd22zD7u4MktdnavogPdzyqpCXyeWnzzgJE0TJSvXR0cap0dHyhoTqtqWyETM3yK9TFjR4eyvEIQlo1lGuSlUdhphuCApVDukXvzfQIRtO6ESK7u/jwMyoh7ckZbmF1qIRfvsqh3r0cYk55FUOQHthyshsCUOGhoQ6VR5mssBCkPI+5oqfwZDLQQkOjJIrLQe8lyFHV9OROKevaoU4iGcNSBsaoPJo+RoNSIA2lPeKU8mhJhkAZqypLFRZhGSE854/48UTD4GqhPxgdc34A0sJzZkuYYa0LOBwPE01GcUeHs7uKM6nyKi+8SMJYLx2dHTsCiHTuZozUVqv/x4i5JlZHdk9BIZ2hyW7kZKYbAleTcstGeAQ5wnOleASgdRd3qS/14WdHhoV08klNJJPQu4898Raa3WUkimH6hIZAheCOPKeGu0ymIXDWEzdXaR5BEXmJYM/oFUPlUL8k7/xib5WNoXCcuB5n1mcQZHhIeg6h3DGVOh6neh2+DEOQUh7tP6g8XFfhHo5ap5WBYAwhBG5bzYgB9v6ID3dsWIWGxjBOM2utWvgpaVKx/v09gXQ+Ixwo3mvSer4KDcXzTB5ElY4KAZ7AAfDOU81o4yAlM5HrEUB2T0Ew2xBMlQQ1zHRDACpPkOMR6InCvkBEnaCLKY9mogvP9eyB4f7ChqB5jYoVZg7O8B2F+DA747PKb7qZLqEhUO+hHrOejB4CHSGIuuYyV/Qyu6C8xKAmL1FhQwAj8gRZCqRS60YfUTGkPIJyx1SmjqElYP3D6RGQKcE5vWKoSFJclbhqUtR2T1aOIJqIMpwI40kkU6GhcuWnM3FZ1cVOGHUyP9ATTCmPuocH8ieKdRacpzqyj72U9+4j/SFmuR2Y+9rGnSiG9Pkj75S5zJ4C3SPQZL6nSoIaDEOgKod6dmeVc9otZmocFhUaCg+qMsaSPAItNKS7obkVQzr6EPbM8JAWJ96fnF1+aCjlEQwpj0CYJma+72Sgn2QtVelpX5OEraGVVVUDnL+0gNHXw36VDA2lJprlNwSDwzFVLRQdGmkItBxC7RhPsB6tHNqXIUWdzhEcTAumFSCzxNVt9+A3W1JVQ6mTtJYsHq8hMJvM1GAimAxTY7fk8QiKGILW81Qyee/f8959tD/E/Fq7Np5yfPkBSGtFjZhpAtk9Be0vqJJ0bU5FMKJyBNVGjmAKaF6lrmJyyjlTtcCldBXr6DITB59Q7e55VBuB9JVuZnhIKx3dL2fTVG5oSJ9SFg1qXcU1k5dkrTR6dVbLmvwTwCYQS10r8009zK0tEBootZmsHPT5xb25hkDv3I2le12asg3BQCiGxSTGXHeeUiDVTqi68qjXltFMVoTaajWWMRJPKI/AYkn1EejyEp5kElkBjwDALaz4kjEWNbnY3xPINjbFDEFVLSx8Ley8P++o2CP9IdbX+FU4cowaQ5lYzCa8TuvIZLGO3lOw/9GsasSUR2DkCKYA/cuV009Q77IrTfGU8mgJhqC6EZBKg37BuYVPxq4mdVWZ2WHcu4eozUM/NeNMFg8pGYuTFf1qezLzAzre+doQkcH89wcnwBCYTNmDbPSlVOmhoWiGIViZtY/qKraVV2qcQUp4TpOiTimPIlQopUiiGDK8Fq1yyJ8RGsqUlwjb64nEk+PKEQC4zXb8Msbixmr2dqU9AmUIRtGjWvUmNQQoJzcXjiXo8kdYY+tUG8ZZOqpTV20rbAj0ngLI7iGIGqGhqaNpBSDyVg71BaKlCc7p6CeIWKhwWEinZS10ZXoEbQw4FwJiDIYgJ1l8siaKId2PMZn5AR2thBTf0fz3l6IzNBbql4yQo06JugVjKoRYuzAdAtQYDEXHXDEEI6WoU6GWqKZIOoohSMtMaAqkJpEODekeQSJJH+o49eP0CDyWKnwk2TDfS89QhAN9PaljjJCXyGXFG9RV+M5s8YPUwHra1YZxlo7qqIhC/uQ0oMJDMEKCGoxk8dRgq1YhnDy9BL2BaGk6QzqZJ4hCiWKdljXQvTst+NW7l07rfCyajG1ZZM4tPlkF53TmblBXS4WkmyeSWq15rXukIi2gDIHJUlxqZCzkmV+sy0b4QmGloJmTHwDY1x1gQd3YK1ysJivVmPAlVCVOSoJaV+sczSOoyvYIhgQkIqpqKOUR2Gro1+zKWHMZOm5LNX6TiY0LVVXXjq4uTAiqZR7BuVyqG1T1UE54SC8dbYkeUd/fPPOZx0JRjwBUT0FVXVY5cEjLETiNHMEU0bwajr2SJSFdX22jPxghGSzDEOhXs8760WONzWtV52bvXpWHCPZwWMyhscY+tjF1uhR12HdyCs7pVDfAu/9YuC58ImlZp76YW3+d//5S5SXKRZ9fPHg4tanGoRRIl7X9WG1f/eashwQjcfb1BFg7d3xhQI+wpqSoU8qjwQE11MVTfDxI7kwCgKGYCjOlPIKqxtQJsa567N4LpGcSzLZHWNrk4kBfN25hwWStLu3iZ+VVyvPKkJ4/3KcMgTtwoCKJYp26antxQ2B1wI2PwYWfTW0KGDmCKWbNW8F3BLbdk9pU77KRlBDx96ir7VKqcHSPYP45oydrM6UmtIqhvWOpGNLRpajHMbh+xmMyw6nvVkm8gcMj7w92VzY/oKOXkGZ0GJtMgnMcRzjnyA9h9VvUZzSDnR1+pIR14zUEZjs+qSp/UqGhoS7lJY+SrNcbp7IUSOPZHoHL1ZQqoxx3jsDmwW8yIYcH2bi8ke7gIDWYlDdQSp5k5RsBATsfSG060h/CZTdj6W+rSKJYp67aykAoluqAzktta5YWVTASp8pqxjzJ84oBJt/0nIisuhpm3Q6P3QKrrwaLPSUzEfN3U5XjDQQicW57aC89QxESSUk8mdR+S250Xkj9nLcy6keqfgmY7Sr+m1RXAtvDzTQ1lVkxpKNLUZ/soaGp5tTr4Imvwis/z7paAyrfVayTb35xNMit3IHPUk/dld8ccaJ7tV2daNfMGachsDjVTIJkIm0IfMdL0uNPT1KLMnuWZghi6aqhmiSY3S2pMkq90WrMa62qJS4EoWAPG5efwi8OhXDFEyMH0hSipgXmn63CQxv/E1A5glO8YYTPX7FEMSiPIJGU+IZjJYfEglOkMwSGR6AwmeCiz6l67ZfuAqBB++clAr0j5CW++Y+9/OSZg2w75mNP1xCH+0J0+MIMBKO8b+hf+PbR1tGPabao0tXObSo8ZLaxLegeu0egzy0+2ZPFU413Hiy5GF75BSTi2fcFKqwzpJOaX5xhCP7+X8yRHXyv9j/yxq23tQ/S4nbQVI5ceR48lmrVCBbxpw1Bf2HV0axl28zYLSZ8oVg68ZxQCVJfxIc7mYAa1VVsNglqylHUzYPboS7I/MFONrTWYrYMUxOLjhpGTCQldz97iO8+vo/n7OdD9w7+8uiT/OGldnZ3DrHBqRUBVChRDOnEeH++prICKOXRyc8PgOERpFl8IbReAE98Dda/K+UREOqHunSt764OP3c/d4hrzpzPl9+8dsTT/OtvXuHpfX1IKUcv62teA7v/AtYqknWL6TuaHPsX2+ZS3crJ2MmdIzgROP0G+O210PYPWHGF2pZMavISFZiOl4/6JWlDsOvP8PLd/M39DjbJlXl333bMN+78AGhS1JoCqS/io8rswB4NlGQIQIV7BkLRdGhIMwT+cD/uREI1k3XGqHVax5b7ylyrpkDqD/Uwy2LGYY9QG4wgXS0Ue+aHdnbyuQdUMcgs5vOcA3Y88gu+m1BDFk+ZrRuC8XcV6+g9E/3BKItL/MgEI/EpyQ+A4RGkEQIuvlmViz7/vdQ/0hJO6wxJKfmf+7fjdlj490vzu5FnL6qnNxBhf08w7/1ZtKxVUhRHniPsVXHisiaTZWLXDAEYoaHxsuwydeX/8t3pbeFBZWQnIjQEan5x3z4Y6oQHPgazTuGx2R8YOaUMGArHONAbZO04w0KgSVFrCqS+qA+3RdNZKtEQeJ0qFp7qSZBRkBLfcB+epGYIgpFxN5MBuKtVZZBPk8QwmUN4k3H6zMULOX656QizPA52fP4yHrz5GmKzTudf5+ziiX/fyMP/9lo21vWr3psKentpmZrSPYLAFI2pBMMQZDN3A6y4Ep65g1r8CAG26EDKEPzx5WO8eGiAm16/omDc7+xFat/nD/TlvT8LPWEc9uGrVjHZcSWLdY0ex0ncUHYiYLYqcbC2f4BPM64T0VWcSf0SJVh4z3vVEJW3/BiX05lXr2bHcZUorohH4KglKQTBYDeDkUG8QqvsKcMQ+EKxVJeyXwCJqBKc03SGBoKxcSeKAdyaAJ4/PEBSJokkg7iTSV71FVaLPdwX5Km2Xt55xnyq7RbcDivWNW/C1v0qC0w9LGlyYerdq8JCFezGHyFcWQJqFsHUhIYMQ5DLhf8PYkEsz95OY5XAngiCsx7fcIyv/G0X6+d5efvp8wo+fEG9kxa3ozRDkFEb3mNXzUzjCg3pGB7B+Dnt3WpIyZZfqtsT0VWciV45dPgZuOwWaFyG12nNViDV2H5MGfyKeARVKuw5GOhUyqMSNS/DU/gznokeGrKarVQJiwozRYP4Y4HUUJr+0PjlJdRa1UWWLzJIMBYkicSTSPJMV+Gy1F+9cASzSfCOMzJez8qr1G+9eqhnT0UTxZAZGirSVJZDMBLHaXgEJwhNK+CUd8ELP+Ish9Zt6Kzntof20heMcsvVa4rGOoUQnL2ojucP9KdnHhfC4Ul1sx4xqZrtcYWGUn8bOYJxU7cIFm2El3+m+ksmqqtYR69hX3Y5bHgfkG7Y8oezk9avtvuY7XHQMMZB8Jl4tFCXP9StlEfjMZUwt5R24vY6bam5CG5LlSrvjATwx4dTgnMDFdAZgrQkhj86lKUz9NhxC8PRxIj9I/EE92xu5+KVTbRkKsrWLVQSJrsegOEBZeQrmCgGJVzpslvyS1EXIBCJ4zJyBCcQG28CJB+PqcmZRyNV/Oy5Q1x31oKSyvXKyxMoKYX9yVkIMY42/EyPwKgaqgynXa/kJvY/lqE8OkHJ4oZl8Nb/gzd/PxWi0MOPgznhoe0VShQDePW4e7hP5QiioVFVR7Mer81WllKqUlSTiVC4nzhJPFKQtHsYqJBH4LQ4sUjwxTKUR5NJjsXdeT3wB3d00ReM8q6z8ow7XXmVmlGw/zF1u8IeAZTQXZzDVI2pBMMQ5Mc7D874AEtjewD40Ut+vE4bny6QIM6lrDzB+mvhjA9wPGiiwWXHYh7jv8RuhIYqzoo3qPzQy3ep+L3JWjEJghEIAWvflvX8+pwBXeoZwF/BRDGAx6VKLwfDAypHMOwrOT8ASnI5npQEo4mUBIQ/pIym2+rCF06QlONvJgPlbbsx4U8MpzyCGks1wurg8T3dI/b/1abDzKur4oIleWTFV12tfj/1TfW7gs1kOrVlGAIpJcHo1JWPGoagEBf8G2GT0nF5rlNw0+tX4ClR4KusPMGKK+ANX6d7KDz2RDGkh9OA4RFUCotdiYPt+Ztq/HM1Taq8d6aEg04qPzDXW5FjuKtUj0xHuE8pj0aHyzIE+hoHglE8thp8ZhM+TajRY3en6ugr4REAuIUFXyKS8gi8VQ2cs6iex/f2ZO23rzvA8wf6edeZC/KHchuWqJnlXdvA4kgLDlaQ+jIMQSiaQMqpUR4FwxAUprqBl+ddT0RaaJmzgLedVlx3JZOy8gQa3UORcRqCDAkMm5EjqBinXa86v/c9PHFhoQJkirrpVDJRDOmZBEd1wblEskyPID03QZei9g+rGR5uR32GzlCFDIHJij8ZS+cIqpvZuLyJw30hDvWmQ7G/2nQEq1nw9g1FvrerVB8B9UsnZPZFOaGhqZxOBoYhKErP+o9ycfwObnrLOWU3w5SVJ0A3BOPoEtVDQ9bq1MQjgwrQsDStJDtRieICZJ5kdV5t9zHHW1WxE6vVbKVawpG4EovzJMszBCm57FAUj8OL32TCFxlQz+VsrLgh8Jgd+ImncwTu2Wxcrgy0Hh4KxxL84eV2LlvdUjyhrlcPVThRrKMPtyrlYjCoJbtdRmjoxOONp8zhgc+8ndWzy7/6KidPkEhK+gKRsVcMQdoLMMJClee069XvieoqLkCNw4IQ2TmC7cd84xaay8WDmcNJpRXtScrCk/XyUJsxUtPtqCVsMtETUmEat2sWA8HxjdPMxa1pI/kjPmxS4nDPZUF9Na31zlR46C+vduAbjnFtviRxJk0r4fR/VhPDJoC6ahvReDJ1ki+G7hE4jaqhEw+TSYz5A7yg3sksj4PnSjAEfYEISTmOZjJIewRGorjyrLpK1dU3j5QUmUhMJoGnyprKEfhCMQ71hcYtNJeLR1jpFupk5alqUBLJJZIlRa31JLQHVNm1xz0nVT5ZV4FkMYDH6lLhp6HjeBKJ1ByCjcubeP5AH+FYgl9uOsyixmrOXlRX/MmEgDferjrJJ4BUL0EJ3cVTOZQGDEMwYag8QT2bDvSN6hp2D6mmk8bxhIb08lGjh6DyWKvgE1vh7A9O+qG9VdZUaGj7cRUOqbhHYE5fgHhKbCRL7V+VnqTm1hq+jg53Y5GSqpq5DASjVFnNVI1xrnIubpubIbOJwaHj2ohKpTz62uWNhGNJfvbcIV4+Msi7zpw/5hGelULvLi46qUzDyBFMY85eVEdvIMr+nkDR/br8yi0fX2hISxYboaGJYQKSiaXgcdpSoaFtFU4Up45hTks0eGpLzw8AWM0mauwWBoejuHWPIBbAnUwialoq1lWs49amwx0LHs+aVXzOonrsFhNff3AvNouJt51eenHHRFGnyW6XkjA2cgTTGD1P8NyB/qL76R5Bs7sSHoFhCKYTtU4rPi00tK3dx7y6qlQ4plJ4rOoioiqZxF6/tOzHe6uV16JLUbcTz9AZqqwh8DhUuOdIdDBrVrHDauasRfVEE0muXDur4u/RWNCbQ0vpLjZyBNOY+XUqTzBawrjbr4WGxiMZoM8tNjyCaYW3ypryCF49Nsi6Od6KH8OjFRq4y6wY0klJUWsXIWGT0HSGmugPRiuWKAZwO5XXESKJW8qskt6LVii5jGvPrnxPwFjIlKIejakODRl1hhOInid4qq2n6HyC7qEwtU4rNss47fKp1yl9HINpg9dpYyAYZTAU5Wj/MO86c5RKmDGgX8l7y+whSD2+ypo1rhLAjQls1fSHoixsKGHMa6nHcqZF/9xmR1bI7poz57N6tpvTF4ySJJ4knDYzNospVTlVjKA2uL66QrmUcjE8ggmmlDzBuHsIdN7wdVh55fifx+CEwVNlxR+Os+XoIFD5RDGkwy2eZLIsnSGdWqcNXyhKja0GodVFuE16x3HpoxpLQZ9JAOC2ZhdG2CwmNrSeGEYA1IWg3kswGsFoHLvFNHaJmXFiGIIJppQ8QffQOHsIDKYtep3+s/tVeHHNGHpaRkOXd/aYrNmaVSVSqw2nMQkTNdqsMI+lmkg8QSASH7uQYr611sxJ/20/8edulNpdPJVDacAwBBNOKXmCHn+YxvH0EBhMW/Sk55N7e1hQ7yxZ76ocdCnq3CvsUvE6bfjDMRJJiVuok5nb5mIgqHIblfUIMkJDVXnE5E4w6kr1CCLxKcsPgGEIJpzR+gmSSUlPoEKhIYNph37i3905VPGyUR2vU4VbvI6xhVW8TitSgm84hkczBB57bVpeooIVPDaLnSrte+SZqLGhFUQJz5XSRzB1EtRgGIJJoVieYCAUJZaQ4+sqNpi26MJzUPn+AR2PW9Xce6vHpqVUm9Fd7Dapz7HbWXnBOZ0aqcJPbk1C+0SmrtpeUmexGlw/NYliMAzBpHDOIuXC5ssTVKSHwGDakqnjX6lhNLnU1y/llrlv4Mpz/2tMj08Lz8XwmNXn2FPdXHEJah036oTpdpfXBT0V1LtsBKMJwrHiekNqFoHhEUxr5tVVMdvj4KfPHOTv2zuyZtDqhsBIFhvkw5uRE6i0xlAmb7roVhrGOKUrU2/IbVFdym7X7IoLzul4TOo9KbcLeiootZcgeDIni4UQdUKIh4QQbdrvvOObhBDXa/u0CSGuz3P/A0KI7eNZy4mMEIL/eeNqIrEkH/zFy7z2fx/n+0/sZzAUpVuXlzBCQwZ5qHFYEQIWNlTjdlQ+UVwJUgqkoRgei+oZ8Ljn0heMIkR2eKsS6KWpNXWLK/q8E0HphiAxZdPJYPwNZTcBj0gpbxVC3KTd/s/MHYQQdcDngA2ABF4SQjwgpRzQ7n8LUFyMZxpw+ZoWLl7ZxMO7urn72UPc+rfd3P7wXubVqiloRrLYIB9mk6DWaZuw/EAlSE0pC0XxaN3FHu8iNbWsylrx2niPpQpneBCr8+SoGoLSPIKpkpeA8RuCNwEbtb/vBh4nxxAAlwEPSSn7AYQQDwGXA78WQriAfwNuBH43zrWc8FjMJi5f08Lla1rY3enn7mcPce8rx2h22yumzmgw/fjONacyr8451csoiNthwWwSDIZivPesT1O79afUN66gP/RyRSuGdN6y+gZWHn9uUseGjpVSDIE+r3gqQ0PjPXKzlLJD+7sTyFd2MAc4mnG7XdsG8EXgG0BotAMJIW5EGQzmzz8xtETGw4oWN195yzr+8/IVDI+SSDKY2Zybb/j6CYQQQpOZiFJXv5arLvwyQMUF53ROXXcdp667ruLPOxHo/UE9Q4VLSMOxJMkpnFcMJRgCIcTDQEueu/4784aUUgohigvvZz/vemCxlPKTQojW0faXUv4Q+CHAhg0bSj7OiY7XacM71YswMBgnXqc1a5IaqKvgE9mTmQxq7BacNjMdvnDBfdJDaU7gHIGU8uJC9wkhuoQQs6SUHUKIWUB3nt2OkQ4fAcxFhZDOATYIIQ5p62gSQjwupdyIgYHBSUWt05aapKbTH4xyylzv1CzoBEEIQYvHkZo5ko+plqCG8ZePPgDoVUDXA/fn2edB4FIhRK1WVXQp8KCU8ntSytlSylbgfGCvYQQMDE5OvFXWlKQEqLj3QKiyEtQnKy1uBx2+4YL3B6ZYghrGbwhuBS4RQrQBF2u3EUJsEEL8GEBLEn8ReFH7+YKeODYwMJgeeJ02fBmhoaFInFhCVlRw7mRFeQSFcwSh1HSyEzhHUAwpZR9wUZ7tm4H3Z9z+CfCTIs9zCFgznrUYGBhMHUqBNB0amqhmspORFrcKDSWTEpNpZKVTKjQ0hTkCo7PYwMBg3NRW2whFE0Ti6uo2rTN0YjbBTSazPA7iSUlvAfG5dLL45A0NGRgYGOCpSncXQ6YhMDrmdR2xzgKVQ6HoyZ8jMDAwMMhQIM0xBCfAEPmpZpZH6S8VMgQBbUyl6ySuGjIwMDBI6Q3peQL9d60RGqLZo7yizgIlpEaOwMDAYFrgSQnPKQPQH4xhM5umNO59otBQbcdiEgU9gmAkjs1iwjpF84rBMAQGBgYVoDYlPKeHhiLUVlsRJ4Ee0ERjMgma3Y7ChmCKdYbAMAQGBgYVYGSOIJY1VGem0+JxFAkNTa0ENRiGwMDAoAJU2czYLaZUaGggNDGCcycrLUU8gkAkTvUUJorBMAQGBgYVwpvRVNY/QcqjJyu6RyDlSL3M0BSPqYTxy1CfMMRiMdrb2wmHC4s7Gcw8HA4Hc+fOxWo1qlcmGiU8ly4fNQxBmha3g1A0gT8cT/Vc6AQiiRHbJptpYwja29upqamhtbXVSFAZAEr4rK+vj/b2dhYuXDjVy5n2eJ1WBkMx4okkvmEjR5BJs0c1lXX5wyNO+sFInNmeqZ1QOG1CQ+FwmPr6esMIGKQQQlBfX294iZOEt8rGQCiaqhyqdxmGQGeWdqLPN5cgGJn60NC0MQSAYQQMRmB8JiaP2morA6FYupnM8AhStGgyE10FDIFRPmpgYDAtUFLUUfoCus6QYQh0mtyquzjXI1Dzio3y0WmF2Wxm/fr1qZ9Dhw7x+OOP4/F4UtsuvvhiPvKRj7B+/XpWrVpFVVVV6r577rkn6/nuu+8+du7cWbH1fec732HJkiUIIejt7U1tl1Ly8Y9/nCVLlrBu3TpefvllAPbs2cPpp5/OunXreO655wCIx+NcfPHFhEKjjpk2mGHUOq3EEpL2AfXZMAxBGrvFTH21bUQvQSSeJJGUUzqdDKZRsvhEoKqqii1btmRtO3ToEBdccAF//vOfR+x/6NAhrrzyyhGP0bnvvvu48sorWbVqVUXWd95553HllVeycePGrO1/+9vfaGtro62tjU2bNvGhD32ITZs28YMf/IBvfetbtLa28olPfII//OEPfO973+O6667D6ZzZs2gNRuKtUif+A71BwDAEueQbWXkiSFDDNDUEn//TDnYe91f0OVfNdvO5N66u6HMW49lnn+WBBx7giSee4JZbbuEPf/gDQ0NDfPCDHyQUCrF48WJ+8pOfUFtby8aNGznllFN44okniMfj/OQnP+HMM88c8Zynnnpq3mPdf//9vOc970EIwdlnn83g4CAdHR1YrVZCoRChUAir1crg4CB/+tOf+Pvf/z7RL9/gJMSr6Q3t7w5k3TZQtLgdHM8JDYU05VEjWTyNGB4eToV53vzmN6e2P/XUU6ntX/rSl0p6rnPPPZerrrqK//3f/2XLli0sXryY97znPXz1q1/l1VdfZe3atXz+859P7R8KhdiyZQvf/e53ee9731vWuo8dO8a8efNSt+fOncuxY8f4yEc+wpe//GWuv/56PvOZz/DFL36Rz3zmM5hMxsfGYCT6NLKDvUFcdgt2y9TGvU80insEU/teTUuPYDKv3DPJFxoCCoaGysHn8zE4OMhrX/taAK6//nre/va3p+6/5pprAHjNa16D3+9ncHAQr9c7rmPOnz+fxx9/HIB9+/bR3t7OypUrefe73000GuWLX/wiy5YtG9cxDKYPuhT14b4QLVNcF38i0uJ20B+MEo4lcFjViT+oDaWZ6hyBcWk3TcgtkxRCcNlll7F+/Xre//73F3iUYs6cORw9ejR1u729nTlz5mTt89///d/ccsst3HHHHbz//e/na1/7WpZHYmDg0XIE0UTSmFWcB904dmcMstdnERihIYOC1NTUMDQ0BIDH46G2tpannnoKgJ///Ocp7wDgt7/9LQBPP/00Ho8Hj8fDgw8+yJYtW/jxj39c9DhXXXUVP/vZz5BS8vzzz+PxeJg1a1bq/ieeeILZs2ezdOlSQqEQJpMJk8lkVA4ZZJGZE6gz8gMjaEk1lQ2ntgX16WRGstigEO985zv5wAc+wB133ME999zD3XffnUoWL1q0iJ/+9KepfR0OB6eeeiqxWIyf/OQneZ/vjjvu4Gtf+xqdnZ2sW7eOK664gh//+MdcccUV/PWvf2XJkiU4nc6s55VScsstt6QMzY033si1115LPB7ne9/73sS+AQYnFVaziRq7haFI3JhVnAe9uzizhDQ1ncxm5AimDYFAYMS2jRs3jijX1GltbWX79u0Fn++8884b0Ufw/PPP5933uuuu4/bbby+6vo9//ON8/OMfH7FdCMGdd96Z9zFCCB566KHU7ZUrV6b6DAwMcvE4rZohMDyCXPINsT9RykeN0JCBgUHF0GUljBzBSGocVlx2S5ZHEIqeGDkCwyOYBuiVPQYGU42eJ6g3DEFemt32HI8ggc1swmaZ2mtywyMwMDCoGCmPwBCcy0vuyMpgJI5zinsIwDAEBgYGFUT3CAx5ify0uKuyPIJgdOrHVIJhCAwMDCqI18gRFKXFY6d7KEIiqUZWnggS1GAYAgMDgwqyoM5JldVMU41RPpqPFk8ViaSkN6CayoKRqZegBsMQVJT29nbe9KY3sXTpUhYvXswnPvEJotFo3n2PHz/O2972tlGf84orrmBwcHBM67n55pv5+te/nnf7nDlzWL9+PWvWrOGBBx4Y0/NXmi9/+ctTvQSDcXL1qXN44j82UuMwykfz0ZJTQho4AaaTgWEIKoaUkre85S1cffXVtLW1sXfvXgKBAP/93/89Yt94PM7s2bNHzB/Ix1//+tdxawbl45Of/CRbtmzh97//Pe9973tJJpMlPS6RSFR8LTpjMQQTuR6D8jGbBE01hs5QIXJHVoZOkBzB1K9gIvjbTdC5rbLP2bIWXn9rwbsfffRRHA4H//zP/wyoITW33XYbCxcu5POf/zy/+93v+OMf/0ggECCRSHD33Xdz5ZVXsn37dkKhEDfccAPbt29n+fLlHD9+nDvvvJMNGzbQ2trK5s2bCQQCvP71r+f888/n2WefZc6cOdx///1UVVXxox/9iB/+8IdEo1GWLFnCz3/+85LnBaxcuRKLxUJvby833ngjR48eJRwO84lPfIIbb7wRAJfLxb/8y7/w8MMPc+edd/Loo4/ypz/9ieHhYc4991x+8IMfIIRg48aNnHrqqTz11FMEg0F+9rOf8ZWvfIVt27bxjne8g1tuuQWAX/ziF9xxxx1Eo1HOOussvvvd7/Lf//3fKfXW1atX88tf/jLvfmazecR6/vznP/PAAw9gsVi49NJL83pBBgYnAnpTma5CqkJDU38aNjyCCrFjxw5OP/30rG1ut5v58+ezb98+AF5++WXuuecennjiiaz9vvvd71JbW8vOnTv54he/yEsvvZT3GG1tbXzkIx9hx44deL1e/vCHPwDwlre8hRdffJGtW7eycuVK/u///q/kdW/atAmTyURjYyM/+clPeOmll9i8eTN33HEHfX19AASDQc466yy2bt3K+eefz0c/+lFefPFFtm/fzvDwcJayqs1mY/PmzXzwgx/kTW96E3feeSfbt2/nrrvuoq+vj127dvHb3/6WZ555hi1btmA2m/nlL3/JrbfemlJv/eUvf1lwv9z1rFy5knvvvZcdO3bw6quv8tnPfrbk125gMNnUV9uwmkXKIwhE4lMuQQ3T1SMocuU+lVxyySXU1dWN2P7000/ziU98AoA1a9awbt26vI9fuHAh69evB+D000/n0KFDAGzfvp3PfvazDA4OEggEuOyyy0Zdy2233cYvfvELampq+O1vf4sQgjvuuIN7770XgKNHj9LW1kZ9fT1ms5m3vvWtqcc+9thjfO1rXyMUCtHf38/q1at54xvfCCgBO4C1a9eyevXqlHjdokWLOHr0KE8//TQvvfQSZ5xxBqBmODQ1NY1Y3yOPPFJwv8z1eDweHA4H73vf+7jyyiu58sorR33tBgZThUkLnekeQSgax3kCeARTv4JpwqpVq0bE/P1+P0eOHGHJkiW8/PLLVFdXj+sYdnu6EsNsNjM8rFQMb7jhBu677z5OOeUU7rrrrpI6jT/5yU/y6U9/OnX78ccf5+GHH+a5557D6XSyceNGwmH1YXU4HJjN6qolHA7z4Q9/mM2bNzNv3jxuvvnm1H6ZazSZTFnrNZlMxONxpJRcf/31fOUrXym6vmL7Za7HYrHwwgsv8Mgjj3DPPffwne98h0cffXTU129gMFXM8jjo8A0TiSeIJaRRPjqduOiiiwiFQvzsZz8DVBLzU5/6FDfccMOo8frzzjuP3/3udwDs3LmTbdvKy28MDQ0xa9YsYrFYKnxSLj6fj9raWpxOJ7t37y4obqef9BsaGggEAiUlvDO56KKLuOeee+ju7gagv7+fw4cPA2C1WonFYqPul0kgEMDn83HFFVdw2223sXXr1rLWY2Aw2TR7HHT5IykJ6uopVh6FcRoCIUSdEOIhIUSb9ru2wH7Xa/u0CSGuz9huE0L8UAixVwixWwjx1nyPPxkQQnDvvffy+9//nqVLl7Js2TIcDkdJlTAf/vCH6enpYdWqVXz2s59l9erVeDyeko/9xS9+kbPOOovzzjuPFStWjGn9l19+OfF4nJUrV3LTTTdx9tln593P6/XygQ98gDVr1nDZZZelQjelsmrVKm655RYuvfRS1q1bxyWXXEJHRwegJK7XrVvHtddeW3S/TIaGhrjyyitZt24d559/Pt/85jfLf/EGBpPILLfyCFIS1CeARyCklGN/sBBfA/qllLcKIW4CaqWU/5mzTx2wGdgASOAl4HQp5YAQ4vOAWUr5WSGECaiTUvaOdtwNGzbIzZs3Z23btWsXK1euHPNrmUoSiQSxWAyHw8H+/fu5+OKL2bNnDzab0Z1ZCU7mz4bB9OPHTx3glr/s4rc3ns07fvg83732NK5YO2v0B1YAIcRLUsoNudvHa4reBGzU/r4beBz4z5x9LgMeklL2awt5CLgc+DXwXmAFgJQyCYxqBKYjoVCI173udcRiMaSUfPe73zWMgIHBNEWfVLa/JwhMvQQ1jN8QNEspdX+9E2jOs88c4GjG7XZgjhDCq93+ohBiI7Af+KiUsmucazrpqKmpIdfDMTAwmJ7o3cX7e9QgqxOhfHTUHIEQ4mEhxPY8P2/K3E+qGFM5cSYLMBd4Vkp5GvAcULATSAhxoxBisxBic09PTxmHMTAwMDhx0JvK9nUrQ+A8GTqLpZQXF7pPCNElhJglpewQQswCuvPsdox0+AjUyf9xoA8IAX/Utv8eeF+RdfwQ+CGoHMFo6zYwMDA4EWke4RFMvSEYb/noA4BeBXQ9cH+efR4ELhVC1GpVRZcCD2oexJ9IG4mLgJ15Hm9gYGAwbbBZTDS4bBwbVH1AJ0KOYLyG4FbgEiFEG3CxdhshxAYhxI8BtCTxF4EXtZ8v6IljVGL5ZiHEq8C7gU+Ncz0GBgYGJzwtHgd6weZJL0MtpeyTUl4kpVwqpbxYP8FLKTdLKd+fsd9PpJRLtJ+fZmw/LKV8jZRynfY8R8aznqnGbDanpJ3f/va3EwqFpnpJPP744zz77LPjeo5Dhw7xq1/9qkIrgt27d3POOedgt9tHCMT9/e9/Z/ny5SxZsoRbb01LhVx77bWsW7eOz3zmM6ltt9xyC/fdd1/F1mVgMFnoCWOLSWAzT31f79SvYBqhi6Zt374dm83G97///ZIeF4/HJ2xNYzEEueuptCGoq6vjjjvuyJK4ANVP8ZGPfIS//e1v7Ny5k1//+tfs3LmTV199laqqKl599VVefPFFfD4fHR0dbNq0iauvvrpi6zIwmCz0EtJquwUhxBSvZppqDX31ha+yu393RZ9zRd0K/vPM3BaJwlxwwQW8+uqr/OlPf+KWW24hGo1SX1/PL3/5S5qbm7n55pvZv38/Bw4cYP78+XzlK1/h3e9+N8Ggqi3+zne+w7nnnsvjjz/O5z73ObxeL9u2beOf/umfWLt2Ld/61rcYHh7mvvvuY/HixfT09PDBD36QI0eUU3X77bczZ84cvv/972M2m/nFL37Bt7/9bVasWDFiv/POO2/Een7961+nXstNN93Erl27WL9+Pddffz0f+tCH+NCHPsTmzZuxWCx885vf5HWvex133XUX9957Lz6fj2PHjnHdddfxuc99bsR709TURFNTE3/5y1+ytr/wwgssWbKERYsWAfDOd76T+++/n6uvvprh4WGSySSxWAyz2cz//M//8PnPf768f6KBwQmC7hGcCIlimKaGYKqJx+P87W9/4/LLL+f888/n+eefRwjBj3/8Y772ta/xjW98A1C6Qk8//TRVVVWEQiEeeughHA4HbW1tXHPNNanegq1bt7Jr1y7q6upYtGgR73//+3nhhRf41re+xbe//W1uv/12PvGJT/DJT36S888/nyNHjnDZZZexa9cuPvjBD+JyuVJX3+9617vy7pe7nkxuvfVWvv71r6fkpr/xjW8ghGDbtm3s3r2bSy+9lL179wLqZL59+3acTidnnHEGb3jDG9iwYUQjY16OHTvGvHnzUrfnzp3Lpk2bWLlyJY2NjZx22mm8+93vZt++fSSTSU477bRx/JcMDKaOFo/6jjlPAJ0hmKaGoJwr90qiD1YB5RG8733vY8+ePbzjHe+go6ODaDTKwoULU/tfddVVqZNuLBbjox/9aEp7Xz+xApxxxhkpOefFixdz6aWXAkrq+bHHHgPg4YcfZufOdNGV3+8nEAiMWGOx/TLXU4ynn36aj33sYwCsWLGCBQsWpNZ7ySWXUF9fD6g5CU8//XTJhqAYt99+e+rvN77xjfzgBz/gS1/6Elu3buWSSy7hAx/4wLiPYWAwWegewYlQMQTT1BBMFXqOIJOPfexj/Nu//RtXXXUVjz/+ODfffHPqvkxZ6ttuu43m5ma2bt1KMpnE4UiP+8uVc86Uetbj+clkkueffz7rcfkott94ZbKBEfFOIQR33nknP/rRjwA1enP27Nl5HztnzhyOHk03obe3tzNnzpysfe6//35OP/10AoEA+/fv53e/+x2XXXYZ1157bclT2QwMpho9R3CihIaMZPEE4/P5Uiezu+++u+h+s2bNwmQy8fOf/7zsWbyXXnop3/72t1O3dYNUU1PD0NDQqPsVI/c5LrjggpTc9d69ezly5AjLly8H4KGHHqK/vz+VvzjvvPP4yEc+wpYtW9iyZUtBIwDK82lra+PgwYNEo1F+85vfpAbdgPKabr/9dv7jP/6D4eHhlNFJJBJEo9FRX4eBwYlCOll8YoSGDEMwwdx88828/e1v5/TTT6ehoaHgfh/+8Ie5++67OeWUU9i9e3fZV+d33HEHmzdvZt26daxatSpVsfTGN76Re++9l/Xr1/PUU08V3K8Y69atw2w2c8opp3Dbbbfx4Q9/mGQyydq1a3nHO97BXXfdlfJSzjzzTN761reybt063vrWt+YNC3V2djJ37ly++c1vcssttzB37lz8fj8Wi4XvfOc7XHbZZaxcuZJ/+qd/YvXq1anH3XnnnVx//fU4nU7WrVtHKBRi7dq1nH766Xi93rLeLwODqcRlt1Bjt5wQg+thnDLUU8V0k6GeLtx1111s3ryZ73znO1O9lCyMz4bBicivNh1habOLM1pHjq+dKCZKhtrAwMDAYAy866z5U72EFIYhMKgYN9xwAzfccMNUL8PAwKBMplWO4GQMcxlMLMZnwsBgdKaNIXA4HPT19RlffIMUUkr6+vpGLak1MJjpTJvQ0Ny5c2lvb8cYWmOQicPhYO7cuVO9DAODE5ppYwisVmtW166BgYGBQWlMm9CQgYGBgcHYMAyBgYGBwQzHMAQGBgYGM5yTsrNYCNEDHB7jwxuA3gou52TBeN0zC+N1zyxKfd0LpJSNuRtPSkMwHoQQm/O1WE93jNc9szBe98xivK/bCA0ZGBgYzHAMQ2BgYGAww5mJhuCHU72AKcJ43TML43XPLMb1umdcjsDAwMDAIJuZ6BEYGBgYGGRgGAIDAwODGc6MMQRCiMuFEHuEEPuEEDdN9XomEiHET4QQ3UKI7Rnb6oQQDwkh2rTftVO5xolACDFPCPGYEGKnEGKHEOIT2vZp/dqFEA4hxAtCiK3a6/68tn2hEGKT9pn/rRDCNtVrnQiEEGYhxCtCiD9rt6f96xZCHBJCbBNCbBFCbNa2jflzPiMMgRDCDNwJvB5YBVwjhFg1tauaUO4CLs/ZdhPwiJRyKfCIdnu6EQc+JaVcBZwNfET7P0/31x4BLpRSngKsBy4XQpwNfBW4TUq5BBgA3jd1S5xQPgHsyrg9U17366SU6zP6B8b8OZ8RhgA4E9gnpTwgpYwCvwHeNMVrmjCklE8C/Tmb3wTcrf19N3D1ZK5pMpBSdkgpX9b+HkKdHOYwzV+7VAS0m1btRwIXAvdo26fd6wYQQswF3gD8WLstmAGvuwBj/pzPFEMwBziacbtd2zaTaJZSdmh/dwLNU7mYiUYI0QqcCmxiBrx2LTyyBegGHgL2A4NSyri2y3T9zN8O/AeQ1G7XMzNetwT+IYR4SQhxo7ZtzJ/zaTOPwKB0pJRSCDFt64aFEC7gD8C/Sin96iJRMV1fu5QyAawXQniBe4EVU7uiiUcIcSXQLaV8SQixcYqXM9mcL6U8JoRoAh4SQuzOvLPcz/lM8QiOAfMybs/Vts0kuoQQswC0391TvJ4JQQhhRRmBX0op/6htnhGvHUBKOQg8BpwDeIUQ+sXedPzMnwdcJYQ4hAr3Xgh8i+n/upFSHtN+d6MM/5mM43M+UwzBi8BSrZrABrwTeGCK1zTZPABcr/19PXD/FK5lQtDiw/8H7JJSfjPjrmn92oUQjZongBCiCrgElR95DHibttu0e91Syv+SUs6VUraivtOPSimvZZq/biFEtRCiRv8buBTYzjg+5zOms1gIcQUqnmgGfiKl/NLUrmjiEEL8GtiIkqbtAj4H3Af8DpiPkvD+JyllbkL5pEYIcT7wFLCNdMz4M6g8wbR97UKIdajkoBl1cfc7KeUXhBCLUFfKdcArwHVSysjUrXTi0EJDn5ZSXjndX7f2+u7VblqAX0kpvySEqGeMn/MZYwgMDAwMDPIzU0JDBgYGBgYFMAyBgYGBwQzHMAQGBgYGMxzDEBgYGBjMcAxDYGBgYHCCIoR4uyYkmBRClDSTWAiR0MTotgghSiqTNwyBwQlNxod6uxDi90II5wmwpo1CiHPH+RyzhRD3jL5n1mNuEEJ8R/v7g0KI94xnDQYnFtrn6q6czduBtwBPlvFUw5oY3Xop5VWlPMAwBAYnOvqHeg0QBT5YyoMyOksngo1AWYYgdz1SyuNSyrcV2n80pJTfl1L+bKyPNzg5kFLuklLuyd2uaUv9rxDiRSHEq0KIfxnPcQxDYHAy8RSwRAjxRk1v/hUhxMNCiGYAIcTNQoifCyGeAX4uhGgVQjwlhHhZ+zlX22+jEOIJIcT9QogDQohbhRDXCqXpv00IsVjbr1EI8Qfty/aiEOI8Tczug8AnNU/lgnz75VtP5gvR1rZd+/sGIcQfhRB/F0pL/msZ+/2zEGKvEOIFlKQCGc/9ae3vJdr7sFV7nfr6/z3jRKHPKKgWQvxF23e7EOIdE/KfMpho3gf4pJRnAGcAHxBCLNTucwghNgshnhdCXF3KkxmicwYnBdoV9euBvwNPA2drwlrvR6lPfkrbdRVKkGtYCyNdIqUMCyGWAr8G9DjrKcBKlFz3AeDHUsozhRpm8zHgX1G6NbdJKZ8WQswHHpRSrhRCfB8ISCm/rq3tV7n7ac+dtZ5RXuJ6lFpqBNgjhPg2ar7C54HTAR9KOuGVPI/9JXCrlPJeIYQDMAkhLgWWojRoBPCAEOI1QCNwXEr5Bm3tnlHWZTDBCCE2AXbABdQJpSIL8J9SygcLPOxSYJ0QQvcqPaj/90FggSZItwh4VAixTUq5v9gaDENgcKJTlfHFeAqlJbQc+K1Qwlo21Idf54GMk64V+I4QYj2QAJZl7PeiLtkrhNgP/EPbvg14nfb3xcAqkVYvdQulbJpLsf0eKMEIgBoo4tPWsxNYgJIIeVxK2aNt/23Oa0AozZk5Usp7AaSUYW37paiThW44XKgTxVPAN4QQXwX+LKV8qoS1GUwgUsqzICWTcYOU8oYSHiaAj+UzFBmCdAeEEI+jLjAMQ2BwUjMspVyfuUG7Wv6mlPIB7ctzc8bdwYy/P4nSWjoFFQYNZ9yXqT2TzLidJP29MKE8j8zHkXHCp4T9grk7FyBzPQnG/90UwFeklD8YcYcQpwFXALcIIR6RUn5hnMcymHweBD4khHhUShkTQixDqazagJCUMiKEaECFE79W7InAyBEYnJx4SEsLXz/Kfh1SyiTwbpQoWzn8AxUmAkDzLACGgJoS9hsvm4DXCiHqhZLXfnvuDtoktnY9FiyEsGshsQeB9+qeiRBijhCiSQgxG3Wi+AXwv8BpFVqrwQQghHizEKIdJSv+FyGE7gH8GNgJvKzlmn6AunhYCWwWQmxFhRJvlVLuHO04hkdgcDJyM/B7IcQA8CiwsMB+3wX+IFSZ5d8p/epc5+PAnUKIV1HflSdRieI/AfcIId6EMgCF9hsXUsoOIcTNwHPAILClwK7vBn4ghPgCEAPeLqX8hxBiJfCc5pkEgOuAJcD/CiGS2r4fGu86DSqDlPJx4PGcbfeSVhrN3J5EKet+JueuZ4G15R7bUB81MDAwmOEYoSEDAwODGY5hCAwMDAxmOIYhMDAwMJjhGIbAwMDAYIZjGAIDAwODGY5hCAwMDAxmOIYhMDAwMJjh/H97Wk8QMeVwwQAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(np.arange(100000,100050,1), reverse_top10fft[100000:100050], label = \"FFT top-10%\")\n",
+ "plt.plot(np.arange(100000,100050,1), conc2[100000:100050], label = \"Original Parameters\")\n",
+ "plt.plot(np.arange(100000,100050,1), top10_og[100000:100050], label = \"Parameter top-10%\")\n",
+ "plt.title('Parameter Values') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.legend()\n",
+ "plt.draw()\n",
+ "plt.savefig(\"Parameters.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 83,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAjAElEQVR4nO3de5gddZXu8e9LYiAgkARixCTQYYh4kKMIPRCPo0aCIQE1eEQHRiUy0egAjjrOo8HL4HA5E+Y4InjByUgk4VFCwGGIXAwxwoiXYDoQgYBMmgimM4G0JAGU2wms80ethmKzu3vvTvfu7vT7eZ79dNWqX1Wtqu7ea9evalcpIjAzs6Ftt/5OwMzM+p+LgZmZuRiYmZmLgZmZ4WJgZma4GJiZGS4GZjtN0lRJbf2dRyVJ35H05f7OwwYHF4MhTtKDkp6S9EdJj0i6XNIr+zuvrkhqkhSShvfS8vaQtF3SsVWmXSTpmt5YT2/K39P5FbGX7JeI+EREnFfDsh6UdFxf5WqDg4uBAbw7Il4JHAk0A1+qZ2YVBs3fUmURiYingauA0yraDQNOBRY1LrtdS28VbOt7g+Yf2PpeRGwCbgIOlzRa0vWS2iVty+EJHW0l3SrpAkm/AJ4EDpZ0uqT7JD0haYOkj5faT5XUJulzkrZI2izpJEknSPovSVslfaHUfjdJ8yQ9IOlRSUsljcnJP8uf2/OI5s05z1/n+rdJWi7poNLyQtKZktYD66ts/iLgfZL2LMWOp/gfuamrbauU6zqkNP6ST/GS3iVpbR6N/FLSG0rTPi9pU67nfknTOltPd8rrlbR//g63576+LffxFcCBwI9yX34u279H0rpsf6uk/1Fa7pGS7swcr5Z0VWk9Hb/nz0t6GPhejX9L5+e++KOkH0naT9L3JT0uabWkpp7uB6uNi4G9QNJE4ATgToq/je8BB1G8WTwFfLNilg8Dc4G9gYeALcC7gH2A04GLJB1Zav9qYA9gPPAPwL8BHwKOAt4KfFnSpGz7SeAk4O3Aa4BtwLdy2tvy56iIeGVE/ErSLOALwP8GxgK3AVdW5HsScAxwWOW2R8Qvgc05f3n7fhARO2rYtppIehOwEPg4sB/wr8AySbtLOhQ4C/jziNibohg9WO86OvFZoI1i34yj2FcRER8Gfk8eHUbEP0t6LcW++3S2v5GiWIyQNAK4FrgcGJPt3luxrlfntIMo/j5q+Vs6hWJ/jwf+DPhVzjMGuA84pzd2gnUhIvwawi+KN5s/Atsp3tC/DYys0u4IYFtp/Fbg3G6W/R/Ap3J4KsWbwLAc3xsI4JhS+zXASTl8HzCtNO0A4P8Bw4GmnHd4afpNwJzS+G4URywH5XgAx3aT75eAm3N4n5z/TTVuW1tpWgCHlMYvB87P4UuB8yqWdT9F0TuEougcB7yim1wvB57O31vH6/HyfqlY77nAdeW8Kv4GjiuNfxlYWrEvN+V2vi2HVZr+89J6pgLPAnt0kXu1v6Uvlsb/BbipNP5uYG1//6/s6i8fGRgUb8CjIuKgiDgjIp6StKekf5X0kKTHKbpmRmU/eoeN5YVImilpVXZDbKc4yti/1OTRiHguh5/Kn4+Upj8FdJy8Pgi4NrsptlMUh+coPtVWcxBwcan9VkAUnzSr5lvFFcA7JL0GOBl4ICLurHHbanUQ8NmOPHNZE4HXREQrxafxrwBbJC3JXDrz1fy9jYqIUcAbumj7f4FW4Obs5prXRdvXUHwwACAinqfYd+Nz2qbId+lUuV/bozgPA0CNf0uVfwed/V1YH3ExsM58FjiU4pP7PrzYNaNSmxfeECTtDvwQ+CowLt+cbqxoX4+NwMzym11E7BHFeY1qt9rdCHy8ov3IKLp/XpZvNRHxEEX30ocouiwW9XDbngTK5x5eXZHnBRV57hkRV2YOP4iIv6AoGgFc2FXOtYqIJyLisxFxMPAe4O9K5yMq98t/5/qB4gIBioK1iaIrbXzGOkysXF3FeC1/S9bPXAysM3tTfCLbniduu+uzHQHsDrQDOyTNBKbvxPq/A1zQcRJY0tg8L0Cu43ng4Ir2Z0t6fbbfV9L7e7DeRRT99m8Bvp+xerdtLfBXkoZJmkHRBdTh34BPSDpGhb0knShpb0mHSjo2i8/TFPv/+R5sw8vkSetD8k38MYqjrI5lP8JL9+VS4ERJ0yS9guLN/BnglxR9+c8BZ0kanr+To7tZfb1/S9YPXAysM18HRgJ/AFYBP+6qcUQ8AfwtxRvJNuCvgGU7sf6Lc/6bJT2RORyT63oSuAD4RXa1TImIayk+RS/Jroh7gJk9WO8PKU5aroyIzT3ctk9R9HNvBz5IcX6BXFYL8DGKE6jbKLpuPpKTdwfmU+zzh4FXAWf3YBuqmQz8hOL80K+Ab0fELTntn4Av5b78+4i4n+Lo6BuZy7spTjA/GxHPUpxkn5Pb9yHgeopi0ZmvU8ffkvUPvbTrz8ysPpJuB74TEd/r71ys53xkYGZ1kfR2Sa/ObqLZFCeu/Wl/kPO3A82sXodSdJntBWwATu7oUrPBy91EZmbmbiIzMxvE3UT7779/NDU19XcaZmaDxpo1a/4QEWOrTRu0xaCpqYmWlpb+TsPMbNCQ9FBn09xNZGZmLgZmZuZiYGZmuBiYmRkuBmZmhouBmZlRYzGQ9BkVz0O9R9KVkvaQNEnS7ZJaVTwDdUS23T3HW3N6U2k5Z2f8fknHl+IzMtbazUM3zMysD3RbDCSNp7h9b3NEHA4Mo3he6YXARRFxCMWteOfkLHMoHml3CHBRtkPSYTnf64EZwLfzfu/DKJ5tO5Pi2bSnZlszM2uQWruJhgMjJQ2neILTZuBY4JqcvojiYeMAs3KcnD4tH6gxC1gSEc9ExO8o7uN+dL5aI2JD3it9SbY1M7MG6fYbyBGxSdJXgd9TPK3oZooHl2+PiB3ZrI0XnzU7nnwmakTskPQYsF/GV5UWXZ5nY0X8mGq5SJoLzAU48MADu0vdrFc0zbvhheEH55/YbdxsMOq2GEgaTfFJfRLFk42upujmabiIWAAsAGhubvbtVq3PlN/oa4mbDXa13JvoOOB3EdEOIOnfKZ4PO0rS8Dw6mEDxsGzy50SgLbuV9gUeLcU7lOfpLG42KHRWJHzEYINFLcXg98AUSXtSdBNNA1qAW4CTKfr4ZwPXZftlOf6rnP7TiAhJy4AfSPoa8BqKZ7L+GhAwWdIkiiJwCsUzZs0aqi8+9bsryQaLWs4Z3C7pGuAOYAdwJ0VXzQ0UDx8/P2OX5SyXAVdIagW2Ury5ExHrJC0F7s3lnBkRzwFIOgtYTnGl0sKIWNd7m2jWOXf7mBUG7ZPOmpubw7ewtp3VX8XARwnWHyStiYjmatP8DWQzMxu8D7cxG8x8LsEGGh8ZmJmZi4GZmbmbyKzfucvIBgIXA7MBxIXB+ou7iczMzMXAzMzcTWRDkL91bPZyPjIwMzMXAzMzczEwMzN8zsBswPJlptZIPjIwMzMXAzMzczEwMzNqKAaSDpW0tvR6XNKnJY2RtELS+vw5OttL0iWSWiXdJenI0rJmZ/v1kmaX4kdJujvnuUSS+mZzzcysmm6LQUTcHxFHRMQRwFHAk8C1wDxgZURMBlbmOMBMiucbTwbmApcCSBoDnAMcAxwNnNNRQLLNx0rzzeiNjTMzs9rU2000DXggIh4CZgGLMr4IOCmHZwGLo7AKGCXpAOB4YEVEbI2IbcAKYEZO2yciVkXxDM7FpWWZmVkD1FsMTgGuzOFxEbE5hx8GxuXweGBjaZ62jHUVb6sSNzOzBqm5GEgaAbwHuLpyWn6ij17Mq7Mc5kpqkdTS3t7e16szMxsy6vnS2Uzgjoh4JMcfkXRARGzOrp4tGd8ETCzNNyFjm4CpFfFbMz6hSvuXiYgFwAKA5ubmPi8+tuvwzenMulZPN9GpvNhFBLAM6LgiaDZwXSl+Wl5VNAV4LLuTlgPTJY3OE8fTgeU57XFJU/IqotNKyzIzswao6chA0l7AO4GPl8LzgaWS5gAPAR/I+I3ACUArxZVHpwNExFZJ5wGrs925EbE1h88ALgdGAjfly8zMGqSmYhARfwL2q4g9SnF1UWXbAM7sZDkLgYVV4i3A4bXkYmZmvc/fQDYzMxcDMzNzMTAzM1wMzMwMFwMzM8PFwMzMcDEwMzNcDMzMDBcDMzPDxcDMzHAxMDMzXAzMzAwXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM6PGYiBplKRrJP1W0n2S3ixpjKQVktbnz9HZVpIukdQq6S5JR5aWMzvbr5c0uxQ/StLdOc8l+SxkMzNrkFqPDC4GfhwRrwPeCNwHzANWRsRkYGWOA8wEJudrLnApgKQxwDnAMcDRwDkdBSTbfKw034yd2ywzM6tHt8VA0r7A24DLACLi2YjYDswCFmWzRcBJOTwLWByFVcAoSQcAxwMrImJrRGwDVgAzcto+EbEqn5+8uLQsMzNrgFqODCYB7cD3JN0p6buS9gLGRcTmbPMwMC6HxwMbS/O3ZayreFuV+MtImiupRVJLe3t7DambmVktaikGw4EjgUsj4k3An3ixSwiA/EQfvZ/eS0XEgohojojmsWPH9vXqzMyGjFqKQRvQFhG35/g1FMXhkeziIX9uyembgIml+SdkrKv4hCpxMzNrkG6LQUQ8DGyUdGiGpgH3AsuAjiuCZgPX5fAy4LS8qmgK8Fh2Jy0HpksanSeOpwPLc9rjkqbkVUSnlZZlZmYNMLzGdp8Evi9pBLABOJ2ikCyVNAd4CPhAtr0ROAFoBZ7MtkTEVknnAauz3bkRsTWHzwAuB0YCN+XLzMwapKZiEBFrgeYqk6ZVaRvAmZ0sZyGwsEq8BTi8llzMzKz31XpkYGb9qGneDS8MPzj/xH7MxHZVvh2FmZm5GJiZmYuBmZnhYmBmZrgYmJkZLgZmZoaLgZmZ4WJgZma4GJiZGS4GZmaGi4GZmeFiYGZmuBiYmRkuBmZmhouBmZnhYmBmZtRYDCQ9KOluSWsltWRsjKQVktbnz9EZl6RLJLVKukvSkaXlzM726yXNLsWPyuW35rzq7Q01M7PO1XNk8I6IOCIiOh5/OQ9YGRGTgZU5DjATmJyvucClUBQP4BzgGOBo4JyOApJtPlaab0aPt8jMzOq2M91Es4BFObwIOKkUXxyFVcAoSQcAxwMrImJrRGwDVgAzcto+EbEqn5+8uLQsMzNrgFqLQQA3S1ojaW7GxkXE5hx+GBiXw+OBjaV52zLWVbytSvxlJM2V1CKppb29vcbUzcysO8NrbPcXEbFJ0quAFZJ+W54YESEpej+9l4qIBcACgObm5j5fn5nZUFHTkUFEbMqfW4BrKfr8H8kuHvLnlmy+CZhYmn1CxrqKT6gSNzOzBum2GEjaS9LeHcPAdOAeYBnQcUXQbOC6HF4GnJZXFU0BHsvupOXAdEmj88TxdGB5Tntc0pS8iui00rLMzKwBaukmGgdcm1d7Dgd+EBE/lrQaWCppDvAQ8IFsfyNwAtAKPAmcDhARWyWdB6zOdudGxNYcPgO4HBgJ3JQvMzNrkG6LQURsAN5YJf4oMK1KPIAzO1nWQmBhlXgLcHgN+ZqZWR/wN5DNzMzFwMzMXAzMzAwXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM8PFwMzMqP2upWY2QDTNu+GF4Qfnn9iPmdiuxEcGZmbmYmBmZi4GZmaGi4GZmeFiYGZmuBiYmRkuBmZmRh3FQNIwSXdKuj7HJ0m6XVKrpKskjcj47jnemtObSss4O+P3Szq+FJ+RsVZJ83px+8zMrAb1HBl8CrivNH4hcFFEHAJsA+ZkfA6wLeMXZTskHQacArwemAF8OwvMMOBbwEzgMODUbGtmZg1SUzGQNAE4Efhujgs4FrgmmywCTsrhWTlOTp+W7WcBSyLimYj4HdAKHJ2v1ojYEBHPAkuyrZmZNUitRwZfBz4HPJ/j+wHbI2JHjrcB43N4PLARIKc/lu1fiFfM01n8ZSTNldQiqaW9vb3G1M3MrDvdFgNJ7wK2RMSaBuTTpYhYEBHNEdE8duzY/k7HzGyXUcuN6t4CvEfSCcAewD7AxcAoScPz0/8EYFO23wRMBNokDQf2BR4txTuU5+ksbmZmDdDtkUFEnB0REyKiieIE8E8j4oPALcDJ2Ww2cF0OL8txcvpPIyIyfkpebTQJmAz8GlgNTM6rk0bkOpb1ytaZmVlNduYW1p8Hlkg6H7gTuCzjlwFXSGoFtlK8uRMR6yQtBe4FdgBnRsRzAJLOApYDw4CFEbFuJ/IyA156q2cz61pdxSAibgVuzeENFFcCVbZ5Gnh/J/NfAFxQJX4jcGM9uZiZWe/xN5DNzMzFwMzMXAzMzAwXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM8PFwMzMcDEwMzNcDMzMDBcDMzPDxcDMzHAxMDMzXAzMzAwXAzMzw8XAzMyooRhI2kPSryX9RtI6Sf+Y8UmSbpfUKumqfH4x+YzjqzJ+u6Sm0rLOzvj9ko4vxWdkrFXSvD7YTjMz60ItRwbPAMdGxBuBI4AZkqYAFwIXRcQhwDZgTrafA2zL+EXZDkmHUTwP+fXADODbkoZJGgZ8C5gJHAacmm3NzKxBui0GUfhjjr4iXwEcC1yT8UXASTk8K8fJ6dMkKeNLIuKZiPgd0ErxDOWjgdaI2BARzwJLsq2ZmTVITecM8hP8WmALsAJ4ANgeETuySRswPofHAxsBcvpjwH7leMU8ncWr5TFXUouklvb29lpSNzOzGtRUDCLiuYg4AphA8Un+dX2ZVBd5LIiI5ohoHjt2bH+kYGa2S6rraqKI2A7cArwZGCVpeE6aAGzK4U3ARICcvi/waDleMU9ncTMza5BariYaK2lUDo8E3gncR1EUTs5ms4HrcnhZjpPTfxoRkfFT8mqjScBk4NfAamByXp00guIk87Je2DYzM6vR8O6bcACwKK/62Q1YGhHXS7oXWCLpfOBO4LJsfxlwhaRWYCvFmzsRsU7SUuBeYAdwZkQ8ByDpLGA5MAxYGBHrem0LzcysW90Wg4i4C3hTlfgGivMHlfGngfd3sqwLgAuqxG8EbqwhXzMz6wP+BrKZmbkYmJmZi4GZmeFiYGZmuBiYmRkuBmZmhouBmZnhYmBmZtT2DWSzQaFp3g39nYLZoOUjAzMzczEwMzMXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM8PFwMzMqO0ZyBMl3SLpXknrJH0q42MkrZC0Pn+OzrgkXSKpVdJdko4sLWt2tl8vaXYpfpSku3OeSySpLzbWzMyqq+XIYAfw2Yg4DJgCnCnpMGAesDIiJgMrcxxgJsXD7icDc4FLoSgewDnAMRSPyzyno4Bkm4+V5pux85tmZma16rYYRMTmiLgjh58A7gPGA7OARdlsEXBSDs8CFkdhFTBK0gHA8cCKiNgaEduAFcCMnLZPRKyKiAAWl5ZlZmYNUNc5A0lNwJuA24FxEbE5Jz0MjMvh8cDG0mxtGesq3lYlXm39cyW1SGppb2+vJ3UzM+tCzcVA0iuBHwKfjojHy9PyE330cm4vExELIqI5IprHjh3b16szMxsyaioGkl5BUQi+HxH/nuFHsouH/Lkl45uAiaXZJ2Ssq/iEKnEzM2uQWq4mEnAZcF9EfK00aRnQcUXQbOC6Uvy0vKpoCvBYdictB6ZLGp0njqcDy3Pa45Km5LpOKy3LzMwaoJbnGbwF+DBwt6S1GfsCMB9YKmkO8BDwgZx2I3AC0Ao8CZwOEBFbJZ0HrM5250bE1hw+A7gcGAnclC8zM2uQbotBRPwc6Oy6/2lV2gdwZifLWggsrBJvAQ7vLhczM+sb/gaymZm5GJiZmYuBmZnhYmBmZrgYmJkZLgZmZoaLgZmZ4WJgZma4GJiZGS4GZmZGbfcmMrMBqmneDS8MPzj/xH7MxAY7HxmYmZmLgZmZuRiYmRk+Z2CDXLnP3Mx6zkcGZmbmYmBmZrU9A3mhpC2S7inFxkhaIWl9/hydcUm6RFKrpLskHVmaZ3a2Xy9pdil+lKS7c55L8jnIZmbWQLUcGVwOzKiIzQNWRsRkYGWOA8wEJudrLnApFMUDOAc4BjgaOKejgGSbj5Xmq1yXmZn1sVqegfwzSU0V4VnA1BxeBNwKfD7ji/M5yKskjZJ0QLZdERFbASStAGZIuhXYJyJWZXwxcBJw085slO3afNLYrPf19GqicRGxOYcfBsbl8HhgY6ldW8a6irdViVclaS7FEQcHHnhgD1M32zX528i2M3b6BHIeBUQv5FLLuhZERHNENI8dO7YRqzQzGxJ6Wgweye4f8ueWjG8CJpbaTchYV/EJVeJmZtZAPe0mWgbMBubnz+tK8bMkLaE4WfxYRGyWtBz4P6WTxtOBsyNiq6THJU0BbgdOA77Rw5xsF+bzBGZ9q9tiIOlKihPA+0tqo7gqaD6wVNIc4CHgA9n8RuAEoBV4EjgdIN/0zwNWZ7tzO04mA2dQXLE0kuLEsU8em5k1WC1XE53ayaRpVdoGcGYny1kILKwSbwEO7y4PG3p8NGDWOP4GspmZ+UZ1NrD4aKB3+DJTq5ePDMzMzEcG1v98NNC3KvevjxSsGhcDsyHGXUhWjYuB9QsfDZgNLC4GZkOYjxKsg4uBNYyPBgY2F4ahzcXA+pQLwODkwjD0uBiYWZdcGIYGFwPrdT4a2HW5MOy6XAysx/ymP7R19vt3kRicXAysLi4A1h0fPQxOLgZWld/0rTf46GHwcDEYgvxGb/3NRWLgcTHYxfiN3gazvvj7dYGpjYtBH/Obs1n/2tn/wXIx2ZXPh6h4OFn/kzQDuBgYBnw3IuZ31b65uTlaWlp6tC6/QZvZYNJbhUfSmohorjZtQBwZSBoGfAt4J9AGrJa0LCLu7d/MzMz6XyOOSAbKw22OBlojYkNEPAssAWb1c05mZkPGgDgyAMYDG0vjbcAxlY0kzQXm5ugfJd3fw/XtD/yhh/P2JedVH+dVH+dVnwGZly7cqbwO6mzCQCkGNYmIBcCCnV2OpJbO+s36k/Oqj/Oqj/Oqz1DLa6B0E20CJpbGJ2TMzMwaYKAUg9XAZEmTJI0ATgGW9XNOZmZDxoDoJoqIHZLOApZTXFq6MCLW9eEqd7qrqY84r/o4r/o4r/oMqbwGzPcMzMys/wyUbiIzM+tHLgZmZrZrFQNJYyStkLQ+f47upN2PJW2XdH1FfJKk2yW1SroqT2Yjafccb83pTX2U1+xss17S7IztLWlt6fUHSV/PaR+R1F6a9tFG5ZXxWyXdX1r/qzLen/trT0k3SPqtpHWS5pfa172/JM3IbWyVNK/K9E63VdLZGb9f0vG1LrMWPc1L0jslrZF0d/48tjRP1d9nA3NrkvRUaf3fKc1zVObcKukSSWpQTh+s+P97XtIROa1R++ttku6QtEPSyRXTOvvfrH9/RcQu8wL+GZiXw/OACztpNw14N3B9RXwpcEoOfwf4mxw+A/hODp8CXNXbeQFjgA35c3QOj67Sbg3wthz+CPDNvtxfXeUF3Ao0V5mn3/YXsCfwjmwzArgNmNmT/UVxMcMDwMG5rN8Ah9WyrcBh2X53YFIuZ1gty+zjvN4EvCaHDwc2leap+vtsYG5NwD2dLPfXwBRAwE0dv9O+zqmizf8EHuiH/dUEvAFYDJxc4/9m3ftrlzoyoLiFxaIcXgScVK1RRKwEnijHsnIeC1xTZf7ycq8BptX5yaSWvI4HVkTE1ojYBqwAZlTk+FrgVRRvcL2hV/LqZrkN3V8R8WRE3AIQxa1N7qD43kpP1HKblM62dRawJCKeiYjfAa25vN649UqP84qIOyPivzO+Dhgpafc6198nuXW2QEkHAPtExKoo3ukW08n/dh/ndGrO25u6zS0iHoyIu4DnK+at+j/Q0/21qxWDcRGxOYcfBsbVMe9+wPaI2JHjbRS3yYDS7TJy+mPZvjfzqnZLjvEVbTo+sZQvAXufpLskXSNpIvXpjby+l4fIXy798wyI/SVpFMUR4MpSuJ79VcvvpLNt7WzeWpbZnZ3Jq+x9wB0R8UwpVu332cjcJkm6U9J/SnprqX1bN8vsy5w6/CVwZUWsEfur3nl7tL8GxPcM6iHpJ8Crq0z6YnkkIkJSw66bbVBepwAfLo3/CLgyIp6R9HGKTzbHlmfo47w+GBGbJO0N/DBzW1zLjH29vyQNp/jHvSQiNmS42/01VEh6PXAhML0U7vHvs5dsBg6MiEclHQX8R+bZ7yQdAzwZEfeUwv29v3rVoCsGEXFcZ9MkPSLpgIjYnIdKW+pY9KPAKEnD85NB+ZYYHbfLaMs3mX2zfW/mtQmYWhqfQNEn2bGMNwLDI2JNaZ3lHL5L0df+En2ZV0Rsyp9PSPoBxSHvYgbA/qL4Ys76iPh6aZ3d7q8q6+juNimdbWtX8+7srVd2Ji8kTQCuBU6LiAc6Zuji99mQ3PKI95nMYY2kB4DXZvtyV1+9+2yn9lc6hYqjggbur67mnVox7630cH/tat1Ey4COM+qzgetqnTH/EG8BOs7Wl+cvL/dk4KcVXTW9kddyYLqk0SqunpmesQ6nUvHHmG+UHd4D3FdHTjuVl6ThkvbPPF4BvAvo+NTUr/tL0vkU/8yfLs/Qg/1Vy21SOtvWZcApKq5SmQRMpjip1xu3XulxXtl1dgPFCfpfdDTu5vfZqNzGqni2CZIOpthnG7LL8HFJU7Ir5jTq+N/emZwyl92AD1A6X9Dg/dWZqv8DPd5f3Z1hHkwvij6+lcB64CfAmIw3Uzw9raPdbUA78BRFf9rxGT+Y4h+2Fbga2D3je+R4a04/uI/y+utcRytwesUyNgCvq4j9E8VJwN9QFLLXNSovYC+KK5vuyhwuBob19/6i+BQUFG/0a/P10Z7uL+AE4L8orvj4YsbOBd7T3bZSdHk9ANxP6WqOasvswd96j/ICvgT8qbRv1lJclNDp77OBub0v172W4sT/u0vLbKZ4s30A+CZ594S+zimnTQVWVSyvkfvrzynep/5EcbSyrrv3jJ7sL9+OwszMdrluIjMz6wEXAzMzczEwMzMXAzMzw8XAzMxwMbBBQtJz+bX/eyRdLWnPAZDTVEn/q8HrbJLUk+vZzbrkYmCDxVMRcUREHA48C3yilpny26R9ZSpQVzHo43zMeszFwAaj24BDJL1bxb3n75T0E0njACR9RdIVkn4BXJGfpm9TcU/4Ozo+zecn+/+UdJ2kDZLmq7h//a9V3Av+z7LdWEk/lLQ6X29Rcb/7TwCfySOWt1ZrVy2f8oZIWiLpxNL45ZJO7iznink/IumbpfHrJU3N4emSfpXzXi3plRmfL+leFTfr+2rv/Ups0OvJN+b88qvRL+CP+XM4xVfr/4biHu4dX5z8KPAvOfwVim+HjszxPYE9cngy0JLDU4HtwAEUzx3YBPxjTvsU8PUc/gHwFzl8IHBfaT1/X8qxq3Yv5FOxXe8FFuXwCIq7UI7sIucm8p7/VDyfAbg+t2l/4GfAXhn/PPAPFN/svr+0z0b19+/Vr4Hz8iGrDRYjJa3N4duAy4BDgavynkMjgN+V2i+LiKdy+BXAN1U8oeo5ipufdVgdebtsFTdGuznjdwPvyOHjgMP04h2K9+n4pF2hq3blfMpuAi5W8UyBGcDPIuIpSft2kXN3plA8XOcXmcsI4FcUt2V+GrhMxVP+ru90CTbkuBjYYPFURBxRDkj6BvC1iFiW3SNfKU3+U2n4M8AjwBspukafLk0r38v/+dL487z4/7EbMCUiyvOhl9++vqt2f6psDBART0u6leJBJX/JizdD6yrnDjt4aVfvHh2rpHjoyamVM0g6muJJfycDZzFEb+FtL+dzBjaY7cuLt+ad3U27zRHxPMU954fVuZ6bgU92jOSndSielrd3De26cxVwOvBW4Md15PwgcISk3VQ8qOfojK8C3iLpkMxjL0mvzaOUfSPiRopi88Ya87MhwMXABrOvAFdLWgP8oYt23wZmS/oN8Do6+ZTehb8FmvOk6728eCXTj4D3dpxA7qJdd24G3g78JIpHH9aa8y8ousbuBS6huNMnEdFOcT7hSkl3UXQRvY6icF2fsZ8Df1frDrBdn+9aamZmPjIwMzMXAzMzw8XAzMxwMTAzM1wMzMwMFwMzM8PFwMzMgP8PscUqwf5NkxgAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.hist(conc2.numpy(), 100, (-0.1,0.1))\n",
+ "plt.title('Parameter Values Histogram') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter values\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"Parameter_Histogram.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 84,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEWCAYAAACNJFuYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAApM0lEQVR4nO3de7wdVXn/8c+XcIsIJEBMIQEDEqURBSFCrNqmBEPAarBFhFoJiEQrXmurwdqCiBX6s3KpF0qFknjhIpYSBYwBjIoaIAhylXIIlyRyCYT7tYHn98d6Npns2eecfU7ONfm+X6/92jNr1sysmT17nllrZq+tiMDMzKxqo8EugJmZDT0ODmZmVuPgYGZmNQ4OZmZW4+BgZmY1Dg5mZlbj4GDWQ5KmSlo+2OVYn0k6UtLVg12OZpIulzRrsMsxEIZNcJD0VOX1kqRnK+Pv76N1bCrpIkn3SApJU5umS9Ipkh7J1ymSlNO2lrRA0mOSvidpRGW+syT9ZTfrvqdpm56StIOkCVmWavrv8iBtjP+fpBcq42e2WP6Rkl7M6U/kMv6iL/ZbX5N0gqTv9uPyz5V0Uj8uPyQ9nft6haSvVY+HoaivA56kcZJWS3pNi2kXS/pqX62rr0haJOlDTWlr7ZeIODAi5raxrJC0a3+Uc6AMm+AQEa9svID7gHdV0r7Xh6u6Gvgb4IEW02YDBwN7AG8E3gV8OKd9GLgBGAtMAN4DIOktwA4R8d9trLu6Ta+MiD9Upo2qpO+RB2ljf3wP+NfK9I90svzfZP5RwDeB8yWNaqNcQ0oG6aF+7O6R+3oa8NfAMT2ZWdLG/VKqftJc3ohYAVwJfKAp3zbAQUC3J1hrbaCOjaH+BeuWpM0knSbpD/k6TdJmOW2qpOWSPi/p4bw677SWEREvRMRpEXE18GKLLLOAf4uI5Xnw/xtwZE7bGfhZRDwP/BLYJa8WTwU+0XdbvO4i4iXgO8AWwER4eT9+VdJ9kh6UdKakkY15JM2UdGPWOu6SNCPTd5A0X9IqSR2SjqnMc4KkCyXNk/SkpFslTa5M/1xeWT8p6Q5J03K5nwfe16glZd5Fkr4s6VfAM5T9e4+k/ZvW993K+Nsk/Tprc8uy9jQbeD/w2Vz+jyrb8UNJKyXdLekTleWMzNrGo5JuA97cg339e8rxsLuk10i6SqXW+bBKDXNUZT335D65CXha0saS5uT+flLSbZLeU8l/pKRfSTo1t3GppD/J9GWSHlKlCaSzz1jSFsDlwA5au9a6UWX9j+RnuU0uq1GjPVrSfcBVLTZ/Lk3BATgMuC0ibu5q26oq69q4krbWVb6kD0q6PT+jBZJenenK/fNQHrs3S9q9zY+vVVleXq+kXSX9XNLj+XlekOm/yOy/y335vkw/Jr8jq/I7s0NludPzO/C4pG/mchvrqX7OjwAntHks/YOkm1RqsWdLGqvS4vCkpCskje5yYyNi2L2Ae4D9c/hEYDHwKmAM8GvgSzltKrAa+BqwGfBnwNPA69pYx3JgalPa48C+lfHJwJM5fCzw/4CRwK+AdwKfBo7v6TY1pU8AAti4i3nPBU7qZvlHAlfn8Igs7wvAqzLtVGA+sA2wJfAj4Cs5bZ/c9ndQLijGAbvltF9QaiGbA3sCK4H9ctoJwHOUK8URwFeAxTntdcAySq2qsZ2vqcz33abyL6LUGF8PbAxs0rzPqvMBrwaeBA7PvNsCe7baX7lN1wP/DGwK7AIsBQ7I6SdTTvDbADsCtwDLu9jXAeyaw5MotdCjgV1zH25GOVZ/AZzWdAzcmOsYmWnvBXbIMr6PcvxuX/lMVwNH5f49KffRN3Id03MfvLKNz3hq8zYBn6R8t8bn8v4DOK/puJxHucgY2WI/jKQcN2+rpP0G+FSb23Z107o2rixnEfChHJ4JdAB/TDk2vgD8OqcdkJ/tKECZZ/tOPreXl1lJW2u/NK33POAfs/ybN23ny8dAju8HPAzslfvy34Ff5LTtgCeAv8zyfxL4v8p6Gp/zx3P6SNo7lhZTWjLGAQ8BvwXelGW9im7OTYN+ou/Ni7WDw13AQZVpBwD3VD7Y1cAWlekXAv/UxjpaBYcXyZNijk/Mg0C5w88CbqKcTMbnh7E1cGZ+eJ2ewHObngIey9f/NH0xHqu8/r5p3nO7WnbTAfZYHnjPAofmNFG+mK+p5H8LcHcO/wdwaotl7pj7ZMtK2leAc3P4BOCKyrRJwLM5vGsesPsDmzQt9wRaB4cTOzsOmucDjgMu7mRfrLW/gH2B+5ryHAf8Vw4vBWZUps2m++DwBPAo5fg8CdioRb6DgRuatueD3XyONwIzK5/pnZVpb8h1j62kPUIJ2t19xlObtwm4HZhWGd8+j52NWXNc7tJNeb8NnFX5vrx8QdLGtrUbHC4Hjq5M24hSu3w15aT8v8CUVp9Bi2PsGdb+rj1F58FhHuU7P76TY6AaHM6mNP02xl+Z+3ICcASlybcxTZQLp2pwuK+bsrc6lt5fGf8h8K3K+MfJc0xnr2HfrES58ri3Mn5vpjU8GhFPN0+XtFOlCv1Um+t6CtiqMr4V8FQUz0XE7Ih4Y0TMoVylfZ7ShLERpdayr7I5phMHR8SofB3cNG27yrTe3sxbHBGjgNGUK8i3Z/oY4BXA9dk88Rjwk0yHEgTuarG8HYBVEfFkJe1eypVKQ/XezTPA5pI2jogO4FOUE/pDks6vVrM7sayb6VWdlbmVV1OOiccq2/95ylUXlO2srvteurdXRIyOiNdExBci4qWs1p+v0pT2BPBdylVj1VrbKOkIlea8Rrl2b5rnwcrwswAR0Zz2Srr/jFt5NXBxJf/tlIuBsZU83X0mc4H3Stqc0sS0ICIeanPb2vVq4PTKclZRTrDjIuIq4OuU2tRDKg+HbNX5ovhE5Xs2CujqoY3P5nquVWky/WAXedc6T0XEU5TAPY6m4yvK2bv54YDm46KdY6n5OGh1XHRqfQgOf6AcHA07ZVrD6GxTXWt6RNwXa9/kbsetlJvRDXtk2loyACgifkK5mluSH/gSyo3sQZUH5t8CH5D0Jkp191ng9ZUvxtaV/bIMqD11QtnP20jaspK2E7CizXJ8PyLeRvn8AjilMamzWZrGn6ac8Br+qDLcWZlbLWcZ5Qp6VOW1ZUQclNPvpwSbhp06WW53/iXX/YaI2Iry4IM6K1u2m/8n8DFg2zxZ3dJinnZ09xm32ufLgAOb9svmUe631crbiaspJ+uZlO2d24tta1zcdfVZf7ipnCMj4tcAEXFGROxNqbm+FviHbsrcloh4ICKOiYgdKA+kfFOdP6G01nkqz0nbUr4r91NaGhrTVB1vrK5pvJ1jaZ2sD8HhPOALksZI2o7Sbtz8GOQXVR5TfTvlSuAHnS1M5abd5jm6qaTN88OCUo38O5XH9HYAPkNpoqjOvzmlWelTmXQ3MFXSpsBbKU0Ugy4iVlGq/P8c5Qb1fwKnSnoVvPwo4gGZ/WzgKJUbxhvltN0iYhnlHs9Xcj+9kdK23u1jqJJeJ2k/lYcHnqOcuF7KyQ8CE9T9E0k3AodJ2kTlRvchlWnfA/aXdKjKjd1tJe1ZWf4ulbzXAk+q3AweKWmEpN0lNW48XwgcJ2m0pPGUKnlvbEmpfT4uaRzdn6S2oJwAVgJIOopydd1jbXzGDwLbStq6MtuZwJe15ubuGEkze7jexn2JUyjt/j/KSW1vW0SspJxE/yY/mw+yduA/k/L5vD6XtbWk9+bwmyXtK2kTSpB5jjXH2TqR9N48HqA0IQZrH8PVY+w8yndozzzm/wW4JiLuAS4F3iDpYJWb7seydvBrpafHUo+tD8HhJMoV+U3AzZR2/uoz7A9QPrg/UE4YH4nyBEln7qCcqMYBC3K4EfH/g3Jw30y5yrk006o+D3wvIpZX5tmO8iVYDlzc4y3sP6cBB+VJ/XOUm3qLs5p6BeWmMRFxLeWm56mUG4w/Z80+OZzSbvoHyrYdHxFXtLHuzShB9GHKZ/QqSjs/rAnej0j6bRfL+CfKSeJR4IvA9xsTIuI+yo3wz1CuXG9kTa3vbGBSNkP8T0S8SLlo2JMSzB+mBM7GifKLlCaBu4GfUp706o0vUm5IPk45drp8vDkibqM8EfcbysnmDZSHHXqrq8/495QT2NLcLzsAp1OaH38q6UnKDc59e7HeeZTa1gVRnubrzbYdQzkBPkJ5KOHXjQkRcTEl+Jyf23ULcGBO3ooSFB+lfIaPUB4c6QtvBq5RaZaeD3wyIhoXfycAc3NfHprfiX+itP3fTzluD8vyP0y5Of+vWb5JlHPa812su0fHUm8ob06sl1R+xPbdiGiuopmZDUlZY15OuaH8s8Eqx/pQczAzG9YkHSBpVDY5fZ5y/2DxYJbJwcHMbPC9hfJ03cOUnhcOjohnB7NA63WzkpmZ9Y5rDmZmVjOsOveq2m677WLChAmDXQwzs2Hj+uuvfzgiuvrh48uGbXCYMGECS5YsGeximJkNG5La+XU/0GazkqRP58/Db5F0Xv7gaWdJ16j0MnhB/sir8SOyCzL9GkkTKss5LtPvqPz4BkkzMq1D0pwebKuZmfWDboND/vruE8DkiNid0vvjYZQfnZwaEbtSfmBydM5yNKU/o10pP5o6JZczKed7PTCD8lPzESrdWn+D8qOVScDhmdfMzAZJuzekNwZG5k+7X0H5hd9+wEU5fS6lV0Aofag0/sjjImBadj8xEzg/Ip6PiLspv9TcJ18dEbE0Il4Azs+8ZmY2SLoNDtnJ1lcp/cTfT/m59vXAYxGxOrMtZ01PnOPIHgRz+uOUDqZeTm+ap7P0GkmzJS2RtGTlypXtbJ+ZmfVCO81KoylX8jtTupbdgtIsNOAi4qyImBwRk8eMaeuGu5mZ9UI7zUr7U7ozXhkR/0fp4OmtwCit+du+8azppnkF2b1xTt+a0pnUy+lN83SWbmZmg6Sd4HAfMEXSK/LewTTgNuBnrOkieRZwSQ7Pz3Fy+lXZbe98SvfKm0namfKvUNcC1wET8+mnTSk3reev+6aZmVlvdfs7h4i4RtJFlK6wVwM3UP4a71JKF7knZdrZOcvZwHckdVC6Sm50S3urpAspgWU1cGx2lYykj1G6xx4BnBMRtT/QMTOzgTNs+1aaPHly+EdwZmbtk3R9RExuJ++w/YW02VAyYc6lLw/fc/I7B7EkZn3DHe+ZmVmNaw5mvVStLZitbxwczPqYm5hsfeDgYNYDri3YhsL3HMzMrMbBwczMahwczMysxvcczPqRb07bcOWag5mZ1bjmYNYNP6FkGyLXHMzMrMbBwczMahwczMysxsHBzMxqHBzMzKzGwcHMzGr8KKvZAPEP4mw46bbmIOl1km6svJ6Q9ClJ20haKOnOfB+d+SXpDEkdkm6StFdlWbMy/52SZlXS95Z0c85zhiT1z+aamVk7ug0OEXFHROwZEXsCewPPABcDc4ArI2IicGWOAxwITMzXbOBbAJK2AY4H9gX2AY5vBJTMc0xlvhl9sXFmZtY7Pb3nMA24KyLuBWYCczN9LnBwDs8E5kWxGBglaXvgAGBhRKyKiEeBhcCMnLZVRCyOiADmVZZlZmaDoKf3HA4DzsvhsRFxfw4/AIzN4XHAsso8yzOtq/TlLdJrJM2m1EbYaaedelh0s/a5ywzb0LVdc5C0KfBu4AfN0/KKP/qwXC1FxFkRMTkiJo8ZM6a/V2dmtsHqSbPSgcBvI+LBHH8wm4TI94cyfQWwY2W+8ZnWVfr4FulmZjZIehIcDmdNkxLAfKDxxNEs4JJK+hH51NIU4PFsfloATJc0Om9ETwcW5LQnJE3Jp5SOqCzLzMwGQVv3HCRtAbwD+HAl+WTgQklHA/cCh2b6ZcBBQAflyaajACJilaQvAddlvhMjYlUOfxQ4FxgJXJ4vMzMbJG0Fh4h4Gti2Ke0RytNLzXkDOLaT5ZwDnNMifQmweztlMTOz/ufuM8zMrMbdZ5gNAnelYUOdaw5mZlbj4GBmZjUODmZmVuN7DmbJXWaYreGag5mZ1Tg4mJlZjYODmZnVODiYmVmNg4OZmdU4OJiZWY2Dg5mZ1fh3DmaDzP0s2VDkmoOZmdU4OJiZWY2Dg5mZ1bQVHCSNknSRpN9Lul3SWyRtI2mhpDvzfXTmlaQzJHVIuknSXpXlzMr8d0qaVUnfW9LNOc8Z+V/SZmY2SNqtOZwO/CQidgP2AG4H5gBXRsRE4MocBzgQmJiv2cC3ACRtAxwP7AvsAxzfCCiZ55jKfDPWbbPMzGxddPu0kqStgT8FjgSIiBeAFyTNBKZmtrnAIuBzwExgXv6X9OKsdWyfeRdGxKpc7kJghqRFwFYRsTjT5wEHA5f3xQaadcU9sZq11k7NYWdgJfBfkm6Q9G1JWwBjI+L+zPMAMDaHxwHLKvMvz7Su0pe3SK+RNFvSEklLVq5c2UbRzcysN9oJDhsDewHfiog3AU+zpgkJgKwlRN8Xb20RcVZETI6IyWPGjOnv1ZmZbbDaCQ7LgeURcU2OX0QJFg9mcxH5/lBOXwHsWJl/fKZ1lT6+RbqZmQ2SboNDRDwALJP0ukyaBtwGzAcaTxzNAi7J4fnAEfnU0hTg8Wx+WgBMlzQ6b0RPBxbktCckTcmnlI6oLMvMzAZBu91nfBz4nqRNgaXAUZTAcqGko4F7gUMz72XAQUAH8EzmJSJWSfoScF3mO7Fxcxr4KHAuMJJyI9o3o83MBlFbwSEibgQmt5g0rUXeAI7tZDnnAOe0SF8C7N5OWczMrP+54z2zIcSd8NlQ4e4zzMysxsHBzMxqHBzMzKzGwcHMzGocHMzMrMbBwczMavwoq21w3BOrWfdcczAzsxoHBzMzq3FwMDOzGgcHMzOrcXAwM7MaBwczM6txcDAzsxoHBzMzq/GP4MyGKP+3gw0m1xzMzKymreAg6R5JN0u6UdKSTNtG0kJJd+b76EyXpDMkdUi6SdJeleXMyvx3SppVSd87l9+R86qvN9TMzNrXk5rDn0fEnhHR+C/pOcCVETERuDLHAQ4EJuZrNvAtKMEEOB7YF9gHOL4RUDLPMZX5ZvR6i8zMbJ2tyz2HmcDUHJ4LLAI+l+nzIiKAxZJGSdo+8y6MiFUAkhYCMyQtAraKiMWZPg84GLh8HcpmthZ3tmfWM+3WHAL4qaTrJc3OtLERcX8OPwCMzeFxwLLKvMszrav05S3SayTNlrRE0pKVK1e2WXQzM+updmsOb4uIFZJeBSyU9PvqxIgISdH3xVtbRJwFnAUwefLkfl+fmdmGqq2aQ0SsyPeHgIsp9wwezOYi8v2hzL4C2LEy+/hM6yp9fIt0MzMbJN0GB0lbSNqyMQxMB24B5gONJ45mAZfk8HzgiHxqaQrweDY/LQCmSxqdN6KnAwty2hOSpuRTSkdUlmVmZoOgnWalscDF+XTpxsD3I+Inkq4DLpR0NHAvcGjmvww4COgAngGOAoiIVZK+BFyX+U5s3JwGPgqcC4yk3Ij2zWgzs0HUbXCIiKXAHi3SHwGmtUgP4NhOlnUOcE6L9CXA7m2U18zMBoB/IW1mZjUODmZmVuPgYGZmNQ4OZmZW4y67zYYBd99tA801BzMzq3FwMDOzGgcHMzOr8T0HW2+5m26z3nPNwczMahwczMysxsHBzMxqHBzMzKzGwcHMzGocHMzMrMbBwczMahwczMyspu3gIGmEpBsk/TjHd5Z0jaQOSRdI2jTTN8vxjpw+obKM4zL9DkkHVNJnZFqHpDl9uH1mZtYLPak5fBK4vTJ+CnBqROwKPAocnelHA49m+qmZD0mTgMOA1wMzgG9mwBkBfAM4EJgEHJ55zcxskLQVHCSNB94JfDvHBewHXJRZ5gIH5/DMHCenT8v8M4HzI+L5iLgb6AD2yVdHRCyNiBeA8zOvmbUwYc6lL7/M+ku7NYfTgM8CL+X4tsBjEbE6x5cD43J4HLAMIKc/nvlfTm+ap7N0MzMbJN0GB0l/ATwUEdcPQHm6K8tsSUskLVm5cuVgF8fMbL3VTs3hrcC7Jd1DafLZDzgdGCWp0avreGBFDq8AdgTI6VsDj1TTm+bpLL0mIs6KiMkRMXnMmDFtFN3MzHqj2y67I+I44DgASVOBv4+I90v6AXAIJWDMAi7JWebn+G9y+lUREZLmA9+X9DVgB2AicC0gYKKknSlB4TDgr/tqA23D4TZ4s76zLv/n8DngfEknATcAZ2f62cB3JHUAqygneyLiVkkXArcBq4FjI+JFAEkfAxYAI4BzIuLWdSiXmZmtox4Fh4hYBCzK4aWUJ42a8zwHvLeT+b8MfLlF+mXAZT0pi5mZ9R//QtrMzGocHMzMrMbBwczMahwczMysxsHBzMxqHBzMzKzGwcHMzGrW5UdwZjbIqr8Kv+fkdw5iSWx945qDmZnVODiYmVmNg4OZmdX4noMNa+6J1ax/uOZgZmY1Dg5mZlbj4GBmZjUODmZmVuPgYGZmNQ4OZmZW021wkLS5pGsl/U7SrZK+mOk7S7pGUoekCyRtmumb5XhHTp9QWdZxmX6HpAMq6TMyrUPSnH7YTjMz64F2ag7PA/tFxB7AnsAMSVOAU4BTI2JX4FHg6Mx/NPBopp+a+ZA0CTgMeD0wA/impBGSRgDfAA4EJgGHZ14zMxsk3QaHKJ7K0U3yFcB+wEWZPhc4OIdn5jg5fZokZfr5EfF8RNwNdAD75KsjIpZGxAvA+ZnXzMwGSVu/kM6r++uBXSlX+XcBj0XE6syyHBiXw+OAZQARsVrS48C2mb64stjqPMua0vftpByzgdkAO+20UztFN9tguIdW60tt3ZCOiBcjYk9gPOVKf7f+LFQX5TgrIiZHxOQxY8YMRhHMzDYIPXpaKSIeA34GvAUYJalR8xgPrMjhFcCOADl9a+CRanrTPJ2lm5nZIGnnaaUxkkbl8EjgHcDtlCBxSGabBVySw/NznJx+VUREph+WTzPtDEwErgWuAybm00+bUm5az++DbTMzs15q557D9sDcvO+wEXBhRPxY0m3A+ZJOAm4Azs78ZwPfkdQBrKKc7ImIWyVdCNwGrAaOjYgXASR9DFgAjADOiYhb+2wLbb3jnljN+l+3wSEibgLe1CJ9KeX+Q3P6c8B7O1nWl4Evt0i/DLisjfKamdkA8C+kzcysxsHBzMxqHBzMzKzGwcHMzGocHMzMrMbBwczMatrqW8nMhhf3s2TryjUHMzOrcXAwM7MaNyvZsOAuM8wGlmsOZmZW4+BgZmY1Dg5mZlbj4GBmZjUODmZmVuPgYGZmNQ4OZmZW085/SO8o6WeSbpN0q6RPZvo2khZKujPfR2e6JJ0hqUPSTZL2qixrVua/U9KsSvrekm7Oec6QpP7YWLMN0YQ5l671MmtHOzWH1cBnImISMAU4VtIkYA5wZURMBK7McYADgYn5mg18C0owAY4H9qX8vejxjYCSeY6pzDdj3TfNzMx6q53/kL4fuD+Hn5R0OzAOmAlMzWxzgUXA5zJ9XkQEsFjSKEnbZ96FEbEKQNJCYIakRcBWEbE40+cBBwOX98kW2rDlq1yzwdOjew6SJgBvAq4BxmbgAHgAGJvD44BlldmWZ1pX6ctbpJuZ2SBpOzhIeiXwQ+BTEfFEdVrWEqKPy9aqDLMlLZG0ZOXKlf29OjOzDVZbwUHSJpTA8L2I+O9MfjCbi8j3hzJ9BbBjZfbxmdZV+vgW6TURcVZETI6IyWPGjGmn6GZm1gvtPK0k4Gzg9oj4WmXSfKDxxNEs4JJK+hH51NIU4PFsfloATJc0Om9ETwcW5LQnJE3JdR1RWZaZmQ2CdrrsfivwAeBmSTdm2ueBk4ELJR0N3AscmtMuAw4COoBngKMAImKVpC8B12W+Exs3p4GPAucCIyk3on0z2sxsELXztNLVQGe/O5jWIn8Ax3ayrHOAc1qkLwF2764sZmY2MPwLaTMzq/E/wZltYKq/H7nn5HcOYklsKHNwsCHFP3wzGxrcrGRmZjUODmZmVuPgYGZmNQ4OZmZW4+BgZmY1Dg5mZlbjR1nNNmD+zYN1xsHBBp1/22A29LhZyczMahwczMysxsHBzMxqHBzMzKzGwcHMzGr8tJINCj+hNPT4sVarcs3BzMxqug0Oks6R9JCkWypp20haKOnOfB+d6ZJ0hqQOSTdJ2qsyz6zMf6ekWZX0vSXdnPOcIamzvyQ1M7MB0k7N4VxgRlPaHODKiJgIXJnjAAcCE/M1G/gWlGACHA/sC+wDHN8IKJnnmMp8zesyM7MB1m1wiIhfAKuakmcCc3N4LnBwJX1eFIuBUZK2Bw4AFkbEqoh4FFgIzMhpW0XE4ogIYF5lWWZmNkh6e89hbETcn8MPAGNzeBywrJJveaZ1lb68RXpLkmZLWiJpycqVK3tZdDMz6846P60UESEp+qIwbazrLOAsgMmTJw/IOq3v+Aml4cNPLllvaw4PZpMQ+f5Qpq8AdqzkG59pXaWPb5FuZmaDqLfBYT7QeOJoFnBJJf2IfGppCvB4Nj8tAKZLGp03oqcDC3LaE5Km5FNKR1SWZWZmg6TbZiVJ5wFTge0kLac8dXQycKGko4F7gUMz+2XAQUAH8AxwFEBErJL0JeC6zHdiRDRucn+U8kTUSODyfJmZ2SDqNjhExOGdTJrWIm8Ax3aynHOAc1qkLwF2764cNjz5PoPZ8OTuM8ysS745vWFy9xlmZlbj4GBmZjVuVrI+5/sM6y83MW04XHMwM7MaBwczM6txs5L1CTclbXjcxLR+c83BzMxqXHOwXnNtwWz95eBgZuvMTUzrHwcH6xHXFsw2DA4OZtanXItYPzg4WLdcW7DecqAYvhwcrCUHBLMNm4ODvcwBwfqTaxHDi4PDBshBwAZbZ8egg8bQ4eCwHnMQsOHGtYuhw8FhGPJJ3zYE7RznDiD9Z8gEB0kzgNOBEcC3I+LkQS7SgPHJ3qx3HED6z5AIDpJGAN8A3gEsB66TND8ibhvckhU+eZsNX335/d2QAs2QCA7APkBHRCwFkHQ+MBPol+Dgk72Z9cZQOHcMVIAaKsFhHLCsMr4c2Lc5k6TZwOwcfUrSHb1c33bAw72ctz+5XD3jcvWMy9UzQ7JcOmWdyvXqdjMOleDQlog4CzhrXZcjaUlETO6DIvUpl6tnXK6ecbl6ZkMv11D5P4cVwI6V8fGZZmZmg2CoBIfrgImSdpa0KXAYMH+Qy2RmtsEaEs1KEbFa0seABZRHWc+JiFv7cZXr3DTVT1yunnG5esbl6pkNulyKiIFYj5mZDSNDpVnJzMyGEAcHMzOrWa+Cg6RtJC2UdGe+j+4k308kPSbpx03pO0u6RlKHpAvy5jiSNsvxjpw+oZ/KNSvz3ClpVqZtKenGyuthSafltCMlraxM+9BAlSvTF0m6o7L+V2X6YO6vV0i6VNLvJd0q6eRK/h7vL0kzchs7JM1pMb3TbZV0XKbfIemAdpfZjt6WS9I7JF0v6eZ8368yT8vPcwDLNkHSs5X1n1mZZ+8sc4ekMyRpgMr0/qbv30uS9sxpA7W//lTSbyWtlnRI07TOvpvrtL8AiIj15gX8KzAnh+cAp3SSbxrwLuDHTekXAofl8JnA3+bwR4Ezc/gw4IK+LhewDbA030fn8OgW+a4H/jSHjwS+3p/7q6tyAYuAyS3mGbT9BbwC+PPMsynwS+DA3uwvysMRdwG75LJ+B0xqZ1uBSZl/M2DnXM6IdpbZz+V6E7BDDu8OrKjM0/LzHMCyTQBu6WS51wJTAAGXNz7T/i5TU543AHcNwv6aALwRmAcc0uZ3s9f7q/Far2oOlC435ubwXODgVpki4krgyWpaRtb9gItazF9d7kXAtB5G4nbKdQCwMCJWRcSjwEJgRlMZXwu8inLC6wt9Uq5uljug+ysinomInwFExAvAbym/m+mNl7t1yWU1unXprKzVbZ0JnB8Rz0fE3UBHLq+dZfZbuSLihoj4Q6bfCoyUtFkP198vZetsgZK2B7aKiMVRznzz6OS73c9lOjzn7Uvdli0i7omIm4CXmuZt+R3og/0FrGfNSsDYiLg/hx8AxvZg3m2BxyJidY4vp3TrAZXuPXL645m/L8vVqguRcU15Glc01UfM/krSTZIukrQjPdMX5fqvrFL/U+XLNCT2l6RRlBrilZXknuyvdj6Tzra1s3nbWWZ31qVcVX8F/DYinq+ktfo8B7JsO0u6QdLPJb29kn95N8vszzI1vA84ryltIPZXT+dd1/0FDJHfOfSEpCuAP2ox6R+rIxERkgbsOd0BKtdhwAcq4z8CzouI5yV9mHLls191hn4u1/sjYoWkLYEfZtnmtTNjf+8vSRtTvshnRHboSBv7a0Mh6fXAKcD0SnKvP88+cj+wU0Q8Imlv4H+ynINO0r7AMxFxSyV5sPdXvxp2wSEi9u9smqQHJW0fEfdn1eqhHiz6EWCUpI3zyqHahUeje4/ledLZOvP3ZblWAFMr4+MpbZqNZewBbBwR11fWWS3Dtylt9Wvpz3JFxIp8f1LS9ylV5HkMgf1F+aHQnRFxWmWd3e6vFuvorluXzra1q3nXtauYdSkXksYDFwNHRMRdjRm6+DwHpGxZI34+y3C9pLuA12b+atNgT/fZOu2vdBhNtYYB3F9dzTu1ad5FrPv+Ata/ZqX5QOOO/SzgknZnzAPzZ0DjaYDq/NXlHgJc1dS00xflWgBMlzRa5emc6ZnWcDhNB2eeOBveDdzegzKtU7kkbSxpuyzHJsBfAI2rqkHdX5JOony5P1WdoRf7q51uXTrb1vnAYSpPwewMTKTcJOyLrmJ6Xa5saruUcsP/V43M3XyeA1W2MSr/7YKkXSj7bGk2MT4haUo23RxBD77b61KmLMtGwKFU7jcM8P7qTMvvQB/sr6Knd7CH8ovSRnglcCdwBbBNpk+m/LtcI98vgZXAs5T2uAMyfRfKF7gD+AGwWaZvnuMdOX2XfirXB3MdHcBRTctYCuzWlPYVyk3F31EC224DVS5gC8qTUzdlGU4HRgz2/qJcJQXlxH9jvj7U2/0FHAT8L+WJkn/MtBOBd3e3rZQmsruAO6g8LdJqmb041ntVLuALwNOVfXMj5SGHTj/PASzbX+W6b6Q8SPCuyjInU06+dwFfJ3t36O8y5bSpwOKm5Q3k/noz5Tz1NKU2c2t354x13V8R4e4zzMysbn1rVjIzsz7g4GBmZjUODmZmVuPgYGZmNQ4OZmZW4+Bgw4KkF7Obglsk/UDSK4ZAmaZK+pMBXucESb15nt6sRxwcbLh4NiL2jIjdgReAj7QzU/7atb9MBXoUHPq5PGZ9xsHBhqNfArtKepdK3/s3SLpC0lgASSdI+o6kXwHfyavtX6r0if/bxtV+Xvn/XNIlkpZKOlml//5rVfrCf03mGyPph5Kuy9dbVfr7/wjw6azRvL1VvlblqW6IpPMlvbMyfq6kQzorc9O8R0r6emX8x5Km5vB0Sb/JeX8g6ZWZfrKk21Q6H/xq330ktr7xVYwNK3nlfSDwE+BqYEpEhMof93wW+ExmnQS8LSKezSaod0TEc5ImUrohmZz59gD+GFhF+RX6tyNiH0mfBD5O6YLjdODUiLha0k6ULgr+WOXPaJ6KiK9m2b7fnC+XvVZ5mjbpAkrXDJdm9wnTgL+l9MPfWZm720fbUX4JvX9EPC3pc8DfSfoG8B7KL8MbXWmYteTgYMPFSEk35vAvgbOB1wEXqPSZtClwdyX//MqJeBPg6yr/4PUipTO3husiuwdX6ejtp5l+M/DnObw/MElremTeqnEl3qSrfPNbBAYof8Ryusp/KswAfpEBbesuytydKZRg9Kssy6bAbyjdUD8HnK3yL4g/7nQJtsFzcLDh4tmI2LOaIOnfga9FxPxsTjmhMvnpyvCngQcptYSNKCfIhup/GbxUGX+JNd+PjSg1lOp8qN59f1f5nm7ODJA1g0WUP255H2s6d+uqzA2rWbtpePPGKil/AnN48wyS9qHUTg4BPsYG2mW5dc/3HGw425o1XRHP6ibf/RHxEqXP/RE9XM9PKU1MAOTVPJR/E9yyjXzduQA4Cng7pbms3TLfA+wpaSOVPy7aJ9MXA2+VtGuWYwtJr81azNYRcRkl+OzRZvlsA+TgYMPZCcAPJF0PPNxFvm8CsyT9DtiNTq7iu/AJYHLexL2NNU9K/Qh4T+OGdBf5uvNT4M+AK6L8VWS7Zf4VpSntNuAMSk+mRMRKyv9lnyfpJkqT0m6UQPbjTLsa+Lt2d4BteNwrq5mZ1bjmYGZmNQ4OZmZW4+BgZmY1Dg5mZlbj4GBmZjUODmZmVuPgYGZmNf8fMmWWIZN/rYEAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.hist(reverse_top10.numpy(), 100, (-0.1,0.1))\n",
+ "plt.title('Top-10% FFT Reconstructed Parameter Values Histogram') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter values\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"FFT_Histogram.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 85,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEWCAYAAACKSkfIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAfa0lEQVR4nO3dfbxcVX3v8c+XhPAkkGBigIRwoES4aAX1lOClagTKoxisqKEqgeKNtthqpS8JqFURXoVeLWCpYipIoAoErBJ5EAJCBRQl4UkeSjlEKIk8BJIgSIAb+N0/1hrYGc+cM3POnDnnZH3fr9e8Zu+119577TUzv1mz9p61FRGYmVkZNhruApiZWec46JuZFcRB38ysIA76ZmYFcdA3MyuIg76ZWUEc9M1GIEkzJS0f7nLUk3SOpC8Odzls4Bz0h4Gk5yqPVyStrcx/pE37GCfpMkkPSwpJM+uWS9Lpkp7Oj9MlKS/bWtI1ktZI+p6kMZX15kv68372/XDlmJ6QdL6k17XjuIaKpK5cT2PbtL1Nc/3t28uyMyRd1o79tFN+nU6pS1uvXiLikxHx1Sa29bCk/YeqrDZwDvrDICJeV3sA/wMcVkn7Xht3dTPwUeDxXpbNBQ4H9gDeAhwGfCIv+wRwBzAZ6ALeDyDpHcD2EfEfTez7sHx8bwO6gS+0UvD8pTRq3p/1XxYR8QJwCXBUXb4xwJHAgs6VbsPSri/mUo2aD1UJJG0i6UxJv82PMyVtkpfNlLRc0kmSnsotqYa/CiLipYg4MyJuBl7uJcsc4OsRsTwiVgBfB47Oy3YCboiIF4GbgJ1zsDoD+NtWjilv+2rgzZImSLpC0kpJq/P01Mrx3yjpVEm3AM/n/R4j6X5Jz0paJukTlfy1OvmcpCclPSbpcEmHSPpvSasknVTJv5GkeZIeyr9uFkraJi/+WX5ek3+hvCOv85d5/6vzr58dK9sLScdJehB4sJfDXwB8QNLmlbQDSZ+7q/s6tnp5X7tU5tdrlUt6r6Q786+Ln0t6S2XZCZJW5P08IGm/RvvpT3W/kibm13BNruubch1fCEwDfpzr8nM5//sk3Zvz3yjpf1W2+zZJd+QyXirpksp+aq/zCZIeB77b5HvplFwXz0n6saTXK/1y/Z2k2yR1DbQeRjMH/ZHl88DewJ6kFvherN9C3haYCEwhBe35knYd4L7eBNxVmb8rpwHcA+wvaTPgncC9pGB/dUQsa2UnknYADiH9ctgI+C6wIykorAXOrlvlY6RfIVsCjwBPAu8FtgKOAc6Q9LZK/m2BTUl18g/Av5F+3bw9l/2LknbKef+G9Ovm3cD2wGrgX/Oyd+Xn8fkX1y8kzQJOAv4cmET6AryorryHAzOA3euPPSJ+DjyW168e3/cjYl0Tx9YUSW8FziP9Qns98G1gUW5E7Ap8CviTiNiS9KXzcKv7aOB4YDmpbiaT6ioi4mOs/wv2nyS9kVR3n8n5ryJ9KYyTNA74IXA+sE3O9/66fW2bl+1Ien80816aTarvKcAfAb/I62wD3A98qR2VMOpEhB/D+CB9APfP0w8Bh1SWHQg8nKdnAuuALSrLFwJfbGIfy4GZdWkvA7tV5qcDAYgUROcDdwOnAVOB24GtgXNIreJT+jmm54A1pMD9TWCzXvLtCayuzN8InNzPsfwI+HSlTtYCY/L8lvkYZlTyLwUOz9P3A/tVlm0H/D9gLKkbK4CxleVXA8dW5jci/QLZMc8HsG8/5f0CcG2e3iqv/9Ymj215ZVkAu1Tmz6+9BsC3gK/WbesB0pfbLqQvl/2Bjfsp6/nAC/l1qz1+V62Xuv2eDFxeLVdv7+s8/0VgYV1drsjH+a48rcrymyv7mQm8BGzaR9l7ey99vjL/dVKjpTZ/GHBnOz/Lo+Xhlv7Isj0pSNY8ktNqVkfE7+uXS5qmysnhJvf1HCkI1WwFPBfJCxExNyLeEhHzSN06JwEfIX1Y3w3MkHRQH9s/PCLGR8SOEfHXEbFW0uaSvi3pEUm/I315jFflRDHwaHUjkg6WdGvuPlhD+tUwsZLl6YiodV+tzc9PVJavBWonkXcEfpi7F9aQvgReJrVSe7MjcFYl/yrSl+KURuXtxYXAeyRtDxwBPBQRdzR5bM3aETi+Vs68rR1I5196SK3rLwNPSro4l6WRr+XXbXxEjCed72nk/wI9wLW5e2peH3nXe29HxCukupuSl62IHI2z+npdGek8CQBNvpfq3weN3hdFcdAfWX5L+gDXTMtpNRMkbVG/PCL+J9Y/OdyMe0ldSDV75LT15MCuiPgJ8MfAkvzhXELfAaE3xwO7klriW/Fal4oqeV794Cudz/gB8DVgcg5CV9Xlb8WjwMHVoBYRm0Y679DbcLOPAp+oy79ZpG6bPyhvbyLiEVK30EdJXQ0LBnhszwPVcwPb1pXz1Lpybh4RF+UyfD8i/pT03grg9L7K3KyIeDYijo+InYH3AZ+tnC+or5f13tuSRPpiWkHqApuS02p2qN9d3Xwz7yXrhYP+yHIR8AVJkyRNJPVR/3tdnq/kftB3kvqDL220sdynu2meHad0GWHtQ3EB6UM6Jbf8jif9dK+uvympe+czOek3wMzcB7sP0FL/Pqn7ZS3pZOk29N+nOg7YBFgJrJN0MHBAi/usOgc4VflkbK7nWXnZSuAVYOe6/CdKelPOv7WkDw5gvwtI/er7ALWrs1o9tjuBv5A0Jn8Rv7uy7N+AT0qaoWQLSYdK2lLSrpL2zV8yL5Dq/5UBHMMfUDp5vEt+Tz1D+tVU2/YTrF+XC4FDJe0naWPS++1F4OekvvaXgU9JGptfk7362X2r7yXLHPRHllNILei7gV+T+tGr100/Tjr5+FtS8PhkRPxXH9t7gPTBmAJck6drra1vAz/O+7kHuDKnVZ0EfC8illfWmUgKVMtJJ99acSawGfAUcCvwk74yR8SzpBPIC0nH/RfAohb3WXVWXv9aSc/mMszI+3oeOBW4JXeR7B0RPyS1ii/OXQj3AAcPYL8/IJ08vD4iHhvgsX2a1A+9htTN9qPagohYAvwf0onM1aQul6Pz4k1IX9xPkd4/bwBOHMAx9GY6cB2pq/AXwDcj4oa87B9JDZg1kv4+Ih4g/dr5l1yWw0gnel+KiJdIJ7uPzcf3UeAK0pdCI2fSwnvJXqP1u9FspFL6c9W/R8TUfrKajXqSfgmcExHfHe6ybGjc0jezYSfp3ZK2zd07c0jni9x6HwL+Z5uZjQS7krq6tiCdKzqi1hVm7eXuHTOzgrh7x8ysICO6e2fixInR1dU13MUwMxtVli5d+lRETOpt2YgO+l1dXSxZsmS4i2FmNqpIeqTRMnfvmJkVxEHfzKwgDvpmZgVx0DczK4iDvplZQZoK+kq35vu10u3YluS0bSQtlvRgfp6Q0yXpG5J6JN2typ2AJM3J+R/Mf7U2M7MOaqWl/56I2DMiuvP8PNKogdOB6/M8pFEIp+fHXNJdfagMfzqDNGzql2pfFGZm1hmD6d6ZRb4hRH4+vJJ+Qb4D062ku9lsR7r13+KIWBURq4HFQF93XjIzszZrNugHaQzypZLm5rTJlQGRHue1W85NYf1bnS3PaY3SzcysQ5r9R+6fRsQKSW8AFkta78YdERGS2jJyW/5SmQswbdq0dmzSOqxr3pWvTj982qHDWBIzq9dUSz/fQ5SIeJJ0t6S9gCdytw35+cmcfQXr399yak5rlF6/r/kR0R0R3ZMm9Tp0hJmZDVC/QT/fb3PL2jTpPp73kG7tVrsCZw5weZ5eBByVr+LZG3gmdwNdAxwgaUI+gXtATjMzsw5ppntnMvDDfD/tscD3I+Inkm4DFko6FngE+FDOfxVwCOk+nc8DxwBExCpJXwVuy/lOjohVbTsSMzPrV79BPyKWAXv0kv40sF8v6QEc12Bb5wHntV5MMzNrB/8j18ysIA76ZmYFcdA3MyuIg76ZWUEc9M3MCuKgb2ZWEAd9M7OCOOibmRXEQd/MrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgzdwu0axfXfOuHO4imFkT3NI3MyuIg76ZWUEc9M3MCuKgb2ZWEAd9M7OCOOibmRXEQd/MrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgTQd9SWMk3SHpijy/k6RfSuqRdImkcTl9kzzfk5d3VbZxYk5/QNKBbT8aMzPrUyst/U8D91fmTwfOiIhdgNXAsTn9WGB1Tj8j50PS7sBs4E3AQcA3JY0ZXPHNzKwVTQV9SVOBQ4Hv5HkB+wKX5SwLgMPz9Kw8T16+X84/C7g4Il6MiN8APcBebTgGMzNrUrMt/TOBzwGv5PnXA2siYl2eXw5MydNTgEcB8vJncv5X03tZ51WS5kpaImnJypUrmz8SMzPrV7/3yJX0XuDJiFgqaeZQFygi5gPzAbq7u2Oo92cD5/vimo0+zdwYfR/gfZIOATYFtgLOAsZLGptb81OBFTn/CmAHYLmkscDWwNOV9JrqOmZm1gH9du9ExIkRMTUiukgnYn8aER8BbgCOyNnmAJfn6UV5nrz8pxEROX12vrpnJ2A68Ku2HYmZtU3XvCtffdiGpZmWfiMnABdLOgW4Azg3p58LXCipB1hF+qIgIu6VtBC4D1gHHBcRLw9i/2Zm1qKWgn5E3AjcmKeX0cvVNxHxAvDBBuufCpzaaiHNzKw9/I9cM7OCOOibmRXEQd/MrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgDvpmZgUZzCibZlaA6vDKD5926DCWxNrBLX0zs4K4pW8t8U01zEY3t/TNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgDvpmZgVx0DczK4iv07d++dp8sw2HW/pmZgVxS9/MmuZxeEY/t/TNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgDvpmZgVx0DczK0i/QV/SppJ+JekuSfdK+kpO30nSLyX1SLpE0ricvkme78nLuyrbOjGnPyDpwCE7KjMz61Uzf856Edg3Ip6TtDFws6Srgc8CZ0TExZLOAY4FvpWfV0fELpJmA6cDH5a0OzAbeBOwPXCdpDdGxMtDcFw2SB56wWzD1G9LP5Ln8uzG+RHAvsBlOX0BcHienpXnycv3k6ScfnFEvBgRvwF6gL3acRBmZtacpvr0JY2RdCfwJLAYeAhYExHrcpblwJQ8PQV4FCAvfwZ4fTW9l3Wq+5oraYmkJStXrmz5gMzMrLGmgn5EvBwRewJTSa3z3YaqQBExPyK6I6J70qRJQ7UbM7MitXT1TkSsAW4A3gGMl1Q7JzAVWJGnVwA7AOTlWwNPV9N7WcfMzDqgmat3Jkkan6c3A/4MuJ8U/I/I2eYAl+fpRXmevPynERE5fXa+umcnYDrwqzYdh5mZNaGZq3e2AxZIGkP6klgYEVdIug+4WNIpwB3AuTn/ucCFknqAVaQrdoiIeyUtBO4D1gHH+codM7PO6jfoR8TdwFt7SV9GL1ffRMQLwAcbbOtU4NTWi2lmZu3gm6iY2YD4hiqjk4dhMDMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxJds2qs8nLLZhs8tfTOzgjjom5kVxEHfzKwgDvpmZgXxiVwzGzSPwzN6uKVvZlYQB30zs4K4e8fM2spdPSObg37h/Icss7K4e8fMrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgvg6fTMbMv6j1sjjoF8g/yHLrFzu3jEzK4iDvplZQRz0zcwK4qBvZlYQn8g1s47wlTwjg1v6ZmYFcUu/EL5M08ygiaAvaQfgAmAyEMD8iDhL0jbAJUAX8DDwoYhYLUnAWcAhwPPA0RFxe97WHOALedOnRMSC9h6O1TjIm1lvmmnprwOOj4jbJW0JLJW0GDgauD4iTpM0D5gHnAAcDEzPjxnAt4AZ+UviS0A36ctjqaRFEbG63QdlZqOH+/o7q98+/Yh4rNZSj4hngfuBKcAsoNZSXwAcnqdnARdEciswXtJ2wIHA4ohYlQP9YuCgdh6MmZn1raU+fUldwFuBXwKTI+KxvOhxUvcPpC+ERyurLc9pjdLr9zEXmAswbdq0VopnZqOEux+HT9NBX9LrgB8An4mI36Wu+yQiQlK0o0ARMR+YD9Dd3d2WbZbCHyQz609Tl2xK2pgU8L8XEf+Rk5/I3Tbk5ydz+gpgh8rqU3Nao3QzM+uQfoN+vhrnXOD+iPjnyqJFwJw8PQe4vJJ+lJK9gWdyN9A1wAGSJkiaAByQ08zMrEOa6d7ZB/gY8GtJd+a0k4DTgIWSjgUeAT6Ul11Fulyzh3TJ5jEAEbFK0leB23K+kyNiVTsOwsw2DL6SZ+j1G/Qj4mZADRbv10v+AI5rsK3zgPNaKaCZmbWP/5FrZiOSW/1Dw2PvmJkVxEHfzKwg7t4ZhXw9vpkNlFv6ZmYFcUvfzEYVn+AdHLf0zcwK4qBvZlYQd++Y2YjX6OIFd/W0zi19M7OCuKU/SvgyTTNrB7f0zcwK4pb+CObWvZm1m4P+CONAb2ZDyUHfzDZovsJnfQ76ZrZBcHBvjk/kmpkVxC39YVJKq6SU47SRxefGGnNL38ysIG7pjwBulZh1hn95uqVvZlYUt/SHmFsWZiNTqZ9NB/0OcjeOmQ03B/0h4OBuNnpt6L8AHPTbxIHezEYDB30zK15JjTZfvWNmVhC39AehpNaBmW0YHPTNzA2YBvqql9F6ktdBv0X+cJjZaOagb2Y2AI0agCP9F4CDfkWj63PdujezDUW/V+9IOk/Sk5LuqaRtI2mxpAfz84ScLknfkNQj6W5Jb6usMyfnf1DSnKE5HDOz4dU178pXHyNRMy3984GzgQsqafOA6yPiNEnz8vwJwMHA9PyYAXwLmCFpG+BLQDcQwFJJiyJidbsOpN1G6gtmZqPHSPx3b79BPyJ+JqmrLnkWMDNPLwBuJAX9WcAFERHArZLGS9ou510cEasAJC0GDgIuGvwhNGe09r+ZmbXTQP+cNTkiHsvTjwOT8/QU4NFKvuU5rVH6H5A0V9ISSUtWrlw5wOKZmVlvBn0iNyJCUrSjMHl784H5AN3d3W3bbiPuxjGzkgw06D8habuIeCx33zyZ01cAO1TyTc1pK3itO6iWfuMA9900B3QzGylGSv/+QLt3FgG1K3DmAJdX0o/KV/HsDTyTu4GuAQ6QNCFf6XNATjMzK1qnr/bpt6Uv6SJSK32ipOWkq3BOAxZKOhZ4BPhQzn4VcAjQAzwPHAMQEaskfRW4Lec7uXZS18ysNMPZC9HM1TtHNli0Xy95AziuwXbOA85rqXRmZgXpRBeQh1Y2MyuIg76ZWUEc9M3MCuKgb2ZWEAd9M7OCOOibmRXEQd/MrCAO+mZmBXHQNzMriIO+mVlBHPTNzArioG9mVhAHfTOzgjjom5kVxEHfzKwgDvpmZgVx0DczK4iDvplZQRz0zcwK4qBvZlYQB30zs4I46JuZFcRB38ysIA76ZmYFcdA3MyuIg76ZWUEc9M3MCuKgb2ZWEAd9M7OCOOibmRXEQd/MrCAO+mZmBXHQNzMrSMeDvqSDJD0gqUfSvE7v38ysZB0N+pLGAP8KHAzsDhwpafdOlsHMrGSdbunvBfRExLKIeAm4GJjV4TKYmRVrbIf3NwV4tDK/HJhRzSBpLjA3zz4n6YFB7G8i8NQg1h8qRZZLpw941SLraxBcrtaMyHLp9EGVa8dGCzod9PsVEfOB+e3YlqQlEdHdjm21k8vVGperNS5Xa0orV6e7d1YAO1Tmp+Y0MzPrgE4H/duA6ZJ2kjQOmA0s6nAZzMyK1dHunYhYJ+lTwDXAGOC8iLh3CHfZlm6iIeBytcblao3L1ZqiyqWIGIrtmpnZCOR/5JqZFcRB38ysIKMu6EvaRtJiSQ/m5wkN8v1E0hpJV9Sl7yTpl3kYiEvyCWUkbZLne/LyriEq15yc50FJc3LalpLurDyeknRmXna0pJWVZR/vVLly+o152Iza/t+Q04ezvjaXdKWk/5J0r6TTKvkHVF/9DQ/S1/FKOjGnPyDpwGa3OZTlkvRnkpZK+nV+3reyTq+vaYfK1SVpbWXf51TWeXsub4+kb0hSB8v1kbrP4CuS9szLOlFf75J0u6R1ko6oW9boszmw+oqIUfUA/gmYl6fnAac3yLcfcBhwRV36QmB2nj4H+Ks8/dfAOXl6NnBJu8sFbAMsy88T8vSEXvItBd6Vp48Gzh7K+uqrXMCNQHcv6wxbfQGbA+/JecYBNwEHD7S+SBcVPATsnLd3F7B7M8dLGk7kLmATYKe8nTHNbHOIy/VWYPs8/WZgRWWdXl/TDpWrC7inwXZ/BewNCLi69pp2olx1ef4YeKjD9dUFvAW4ADiiyc/mgOpr1LX0ScM2LMjTC4DDe8sUEdcDz1bT8jfhvsBlvaxf3e5lwH4ttjSaKdeBwOKIWBURq4HFwEF1ZXwj8AZSIGuHtpSrn+12tL4i4vmIuAEg0nAet5P+8zFQzQwP0uh4ZwEXR8SLEfEboCdvrx1Djgy4XBFxR0T8NqffC2wmaZMW99/2cjXaoKTtgK0i4tZIEe0CGny2O1CuI/O67dJvuSLi4Yi4G3ilbt1ePwODqa/RGPQnR8RjefpxYHIL674eWBMR6/L8ctLQEFAZIiIvfybnb2e5ehuGYkpdnlrro3pZ1Qck3S3pMkk70Jp2lOu7+WftFysfkBFRX5LGk37RXV9JbrW+mnldGh1vo3Wb2eZQlqvqA8DtEfFiJa2317RT5dpJ0h2S/lPSOyv5l/ezzaEuV82HgYvq0oa6vlpdd8D1NeKGYQCQdB2wbS+LPl+diYiQ1LFrTjtUrtnAxyrzPwYuiogXJX2C1ErZt7rCEJfrIxGxQtKWwA9y2S5oZsWhri9JY0kfzm9ExLKc3G99lUTSm4DTgQMqyQN+TdvgMWBaRDwt6e3Aj3IZRwRJM4DnI+KeSvJw1lfbjcigHxH7N1om6QlJ20XEY/knzpMtbPppYLyksflbvjoMRG2IiOU5mGyd87ezXCuAmZX5qaT+wto29gDGRsTSyj6rZfgOqS98PUNZrohYkZ+flfR90k/VCxgB9UX688qDEXFmZZ/91leD/fQ3PEij4+1r3cEOOTKYciFpKvBD4KiIeKi2Qh+v6ZCXK/+CfTHvf6mkh4A35vzVLrqO11c2m7pWfofqq691Z9ateyODqK/R2L2zCKidwZ4DXN7sivkNdwNQOzteXb+63SOAn9Z1sbSjXNcAB0iaoHS1ygE5reZI6t5wOSDWvA+4v4UyDapcksZKmpjLsTHwXqDWAhrW+pJ0CukD+5nqCgOsr2aGB2l0vIuA2UpXhewETCedYGvHkCMDLlfu9rqSdLL8llrmfl7TTpRrktJ9NZC0M6m+luWuvt9J2jt3nxxFC5/twZYrl2cj4ENU+vM7WF+N9PoZGFR9NXO2dyQ9SP1v1wMPAtcB2+T0buA7lXw3ASuBtaT+rgNz+s6kD2UPcCmwSU7fNM/35OU7D1G5/jLvowc4pm4by4Dd6tL+kXQi7i7SF9ZunSoXsAXpSqK7cxnOAsYMd32RWjVBCuh35sfHB1NfwCHAf5Ousvh8TjsZeF9/x0vqrnoIeIDKFRS9bXMA7/cBlQv4AvD7Sv3cSbpAoOFr2qFyfSDv907SCfjDKtvsJgXUh4CzySMGdKJcedlM4Na67XWqvv6EFKd+T/rlcW9/MWOg9eVhGMzMCjIau3fMzGyAHPTNzArioG9mVhAHfTOzgjjom5kVxEHfRhRJL+e/u98j6VJJm4+AMs2U9L87vM8uSa1eD27WLwd9G2nWRsSeEfFm4CXgk82slP9dOVRmAi0F/SEuj9mAOejbSHYTsIukw5TGPr9D0nWSJgNI+rKkCyXdAlyYW8c3KY1LfnutdZ5b6v8p6XJJyySdpjR++q+UxiP/o5xvkqQfSLotP/ZRGm/9k8Df5V8g7+wtX2/lqR6IpIslHVqZP1/SEY3KXLfu0ZLOrsxfIWlmnj5A0i/yupdKel1OP03SfUoDz32tfS+JjXoD+ZegH34M1QN4Lj+PJf2t/K9I44jX/kj4ceDrefrLpH9LbpbnNwc2zdPTgSV5eiawBtiONO79CuAredmngTPz9PeBP83T04D7K/v5+0oZ+8r3annqjuv9wII8PY40cuJmfZS5izzuPHX3CACuyMc0EfgZsEVOPwH4B9K/nR+o1Nn44X5d/Rg5D/8EtZFmM0l35umbgHOBXYFL8rg644DfVPIvioi1eXpj4GylOx69TBrIq+a2yEM5Kw3ydW1O/zXwnjy9P7C7Xhs5d6tay7lOX/mq5am6GjhLaUz7g4CfRcRaSVv3Ueb+7E26icstuSzjgF+Qhgt+AThX6c5xVzTcghXHQd9GmrURsWc1QdK/AP8cEYtyt8aXK4t/X5n+O+AJYA9S1+ULlWXVseRfqcy/wmufg42AvSOiuh76w+HT+8r3+/rMABHxgqQbSTfF+DCvDerVV5lr1rF+V+ymtV2SbrBxZP0KkvYi3T3uCOBTFDy8tK3Pffo2GmzNa8PGzukn32MR8QppzPMxLe7nWuBvajO59Q3pDmxbNpGvP5cAxwDvBH7SQpkfBvaUtJHSTWH2yum3AvtI2iWXYwtJb8y/OraOiKtIXyp7NFk+K4CDvo0GXwYulbQUeKqPfN8E5ki6C9iNBq3uPvwt0J1Pft7Ha1cO/Rh4f+1Ebh/5+nMt8G7guki3zWu2zLeQurTuA75BGp2SiFhJ6u+/SNLdpK6d3UhfUFfktJuBzzZbAbbh8yibZmYFcUvfzKwgDvpmZgVx0DczK4iDvplZQRz0zcwK4qBvZlYQB30zs4L8f+sJ3rihP1HBAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.hist(top10_og[top10_og.abs() >0].numpy(), 100, (-0.1,0.1))\n",
+ "plt.title('Top-10% Parameter Values Histogram') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter values\")\n",
+ "plt.draw()\n",
+ "plt.savefig(\"top10_Histogram.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 86,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([-0.14400896, -0.04817872, 0.20703338, ..., 0.0729612 ,\n",
+ " -0.06001848, -0.03798665], dtype=float32)"
+ ]
+ },
+ "execution_count": 86,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "top10_og[top10_og.abs() >0].numpy()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Per Layer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 87,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat5000 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat5000.append(v.flatten())\n",
+ "conc500 = torch.cat(flat5000)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 88,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 88,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc500, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 134,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(3.8334)\n",
+ "tensor(0.5911)\n",
+ "tensor(14.2745)\n",
+ "tensor(0.2714)\n",
+ "tensor(29.4115)\n",
+ "tensor(0.4823)\n",
+ "tensor(9.3226)\n",
+ "tensor(0.3447)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "errs = []\n",
+ "lens = []\n",
+ "fft_layers = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " lens.append(len(flat))\n",
+ " flat_fft = fft.rfft(flat)\n",
+ " topk = torch.topk(\n",
+ " flat_fft.abs(), round(0.1*len(flat_fft)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ " top10[topk.indices] = flat_fft[topk.indices]\n",
+ " reverse_top10 = fft.irfft(top10)\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " print(err)\n",
+ " errs.append(err*err)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 135,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[800, 32, 51200, 64, 1605632, 512, 31744, 62]"
+ ]
+ },
+ "execution_count": 135,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lens"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 136,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1171.1743"
+ ]
+ },
+ "execution_count": 136,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 137,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "34.222424"
+ ]
+ },
+ "execution_count": 137,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 93,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fft_conc = torch.cat(fft_layers)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 94,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'conc5000' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
+ "Input \u001b[0;32mIn [94]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m torch\u001b[38;5;241m.\u001b[39mnorm(fft_conc \u001b[38;5;241m-\u001b[39m \u001b[43mconc5000\u001b[49m\n\u001b[1;32m 2\u001b[0m ,\u001b[38;5;241m2\u001b[39m)\n",
+ "\u001b[0;31mNameError\u001b[0m: name 'conc5000' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "torch.norm(fft_conc - conc5000\n",
+ " ,2)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Almost no difference in layerwise vs over the entire weight"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 138,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "dict_keys(['0', '500', '1000', '1500', '2000', '2500', '3000', '3500', '4000', '4500', '5000', '5500', '6000', '6500', '7000', '7500', '8000', '8500', '9000', '9500', '10000', '10500', '11000', '11500', '12000', '12500', '13000', '13500', '14000', '14500', '15000', '15500', '16000', '16500', '17000'])"
+ ]
+ },
+ "execution_count": 138,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "weights.keys()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 139,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(3.8017)\n",
+ "tensor(0.6525)\n",
+ "tensor(9.6627)\n",
+ "tensor(0.2314)\n",
+ "tensor(14.0121)\n",
+ "tensor(0.3602)\n",
+ "tensor(6.0212)\n",
+ "tensor(0.3187)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "errs = []\n",
+ "lens = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " lens.append(len(flat))\n",
+ " topk = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat))\n",
+ " top10[topk.indices] = flat[topk.indices]\n",
+ " err = torch.norm(top10 - flat, 2)\n",
+ " print(err)\n",
+ " errs.append(err*err)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 140,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[800, 32, 51200, 64, 1605632, 512, 31744, 62]"
+ ]
+ },
+ "execution_count": 140,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lens"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 141,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "341.1233"
+ ]
+ },
+ "execution_count": 141,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 142,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "18.469522"
+ ]
+ },
+ "execution_count": 142,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 99,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat5000 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat5000.append(v.flatten())\n",
+ "conc5000 = torch.cat(flat5000)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 100,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 100,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc5000, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 101,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(2.5049)\n",
+ "tensor(0.4643)\n",
+ "tensor(6.6074)\n",
+ "tensor(0.1233)\n",
+ "tensor(12.2971)\n",
+ "tensor(0.1902)\n",
+ "tensor(4.0843)\n",
+ "tensor(0.1482)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "errs = []\n",
+ "errs1 = []\n",
+ "lens = []\n",
+ "fft_layers = []\n",
+ "for v in weights[\"1000\"].values():\n",
+ " flat = v.flatten()\n",
+ " lens.append(len(flat))\n",
+ " flat_fft = fft.rfft(flat)\n",
+ " topk = torch.topk(\n",
+ " flat_fft.abs(), round(0.2*len(flat_fft)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ " top10[topk.indices] = flat_fft[topk.indices]\n",
+ " reverse_top10 = fft.irfft(top10)\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " errs1.append(torch.norm(reverse_top10 - flat, 1))\n",
+ " print(err)\n",
+ " errs.append(err*err)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 102,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[800, 32, 51200, 64, 1605632, 512, 31744, 62]"
+ ]
+ },
+ "execution_count": 102,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lens"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 103,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "218.12065"
+ ]
+ },
+ "execution_count": 103,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 104,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "14.7689085"
+ ]
+ },
+ "execution_count": 104,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 105,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fft_conc = torch.cat(fft_layers)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 106,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(39.6843)"
+ ]
+ },
+ "execution_count": 106,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(fft_conc - conc5000,2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 107,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[tensor(57.1960),\n",
+ " tensor(2.2182),\n",
+ " tensor(1148.8655),\n",
+ " tensor(0.7960),\n",
+ " tensor(12109.6943),\n",
+ " tensor(3.3738),\n",
+ " tensor(578.7853),\n",
+ " tensor(0.9163)]"
+ ]
+ },
+ "execution_count": 107,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "errs1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 108,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "13901.846"
+ ]
+ },
+ "execution_count": 108,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 109,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(3.8017)\n",
+ "tensor(0.6525)\n",
+ "tensor(9.6627)\n",
+ "tensor(0.2314)\n",
+ "tensor(14.0121)\n",
+ "tensor(0.3602)\n",
+ "tensor(6.0212)\n",
+ "tensor(0.3187)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "errs = []\n",
+ "lens = []\n",
+ "errs1 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " lens.append(len(flat))\n",
+ " topk = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat))\n",
+ " top10[topk.indices] = flat[topk.indices]\n",
+ " err = torch.norm(top10 - flat, 2)\n",
+ " print(err)\n",
+ " errs.append(err*err)\n",
+ " errs1.append(torch.norm(top10 - flat, 1))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 110,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[800, 32, 51200, 64, 1605632, 512, 31744, 62]"
+ ]
+ },
+ "execution_count": 110,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lens"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 111,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "341.1233"
+ ]
+ },
+ "execution_count": 111,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 112,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "18.469522"
+ ]
+ },
+ "execution_count": 112,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 113,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[tensor(86.4496),\n",
+ " tensor(3.1551),\n",
+ " tensor(1531.9570),\n",
+ " tensor(1.5015),\n",
+ " tensor(14199.5361),\n",
+ " tensor(6.1879),\n",
+ " tensor(814.4448),\n",
+ " tensor(2.0511)]"
+ ]
+ },
+ "execution_count": 113,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "errs1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 114,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "16645.283"
+ ]
+ },
+ "execution_count": 114,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sum(errs1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 115,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n",
+ "None\n"
+ ]
+ }
+ ],
+ "source": [
+ "flat5000 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " print(v.grad)\n",
+ " flat5000.append(v.flatten())\n",
+ "conc5000 = torch.cat(flat5000)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Wavelets <a class=\"anchor\" id=\"wt\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 116,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: PyWavelets in /home/jeffrey/anaconda3/envs/sacs39/lib/python3.9/site-packages (1.2.0)\n",
+ "Requirement already satisfied: numpy>=1.17.3 in /home/jeffrey/anaconda3/envs/sacs39/lib/python3.9/site-packages (from PyWavelets) (1.22.3)\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install PyWavelets"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 117,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pywt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 118,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#(cA, cD) = pywt.dwt(, 'db1')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# pywt.wavelist(kind='discrete', )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 143,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 18.427034\n",
+ "db1 18.427034\n",
+ "sym2 18.36348\n",
+ "coif1 18.393574\n",
+ "bior1.1 18.427034\n",
+ "rbio1.1 18.427034\n",
+ "dmey 18.671127\n",
+ "bior4.4 18.496372\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "for wavelet in wavelets:\n",
+ " errs = []\n",
+ " errs1 = []\n",
+ " lens = []\n",
+ " fft_layers = []\n",
+ " for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " #print(flat.shape)\n",
+ " lens.append(len(flat))\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " errs1.append(torch.norm(reverse_top10 - flat, 1))\n",
+ " #print(err)\n",
+ " errs.append(err*err)\n",
+ " # print(flat[0:10])\n",
+ " # print(reverse_top10[0:10])\n",
+ " print(wavelet, np.sqrt(np.sum(errs)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 156,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "bior1.1 15.07145881652832 16107.921875\n",
+ "bior1.3 15.25814437866211 16279.986328125\n",
+ "bior1.5 15.425777435302734 16436.12109375\n",
+ "bior2.2 15.66141128540039 16473.904296875\n",
+ "bior2.4 15.516386985778809 16417.8203125\n",
+ "bior2.6 15.544709205627441 16466.162109375\n",
+ "bior2.8 15.579778671264648 16503.138671875\n",
+ "bior3.1 25.344850540161133 24354.4453125\n",
+ "bior3.3 18.38104248046875 18552.30859375\n",
+ "bior3.5 17.479848861694336 17874.1015625\n",
+ "bior3.7 17.260520935058594 17713.20703125\n",
+ "bior3.9 17.19255828857422 17672.990234375\n",
+ "bior4.4 15.1978120803833 16241.7060546875\n",
+ "bior5.5 15.467646598815918 16527.41796875\n",
+ "bior6.8 15.204909324645996 16253.8125\n",
+ "coif1 15.142683029174805 16156.6630859375\n",
+ "coif2 15.175430297851562 16219.298828125\n",
+ "coif3 15.218149185180664 16275.017578125\n",
+ "coif4 15.248283386230469 16304.376953125\n",
+ "coif5 15.278726577758789 16337.1923828125\n",
+ "coif6 15.300649642944336 16357.455078125\n",
+ "coif7 15.324337005615234 16380.2119140625\n",
+ "coif8 15.34239387512207 16396.9453125\n",
+ "coif9 15.35299301147461 16408.88671875\n",
+ "coif10 15.358224868774414 16412.31640625\n",
+ "coif11 15.375347137451172 16429.083984375\n",
+ "coif12 15.383316993713379 16440.47265625\n",
+ "coif13 15.401575088500977 16450.142578125\n",
+ "coif14 15.413949012756348 16466.5859375\n",
+ "coif15 15.430389404296875 16478.208984375\n",
+ "coif16 15.438526153564453 16489.732421875\n",
+ "coif17 15.44447135925293 16493.5625\n",
+ "db1 15.07145881652832 16107.921875\n",
+ "db2 15.11799430847168 16146.3642578125\n",
+ "db3 15.206748008728027 16251.126953125\n",
+ "db4 15.276558876037598 16337.6650390625\n",
+ "db5 15.346190452575684 16396.228515625\n",
+ "db6 15.424012184143066 16471.95703125\n",
+ "db7 15.465736389160156 16520.9609375\n",
+ "db8 15.5084228515625 16558.216796875\n",
+ "db9 15.579204559326172 16622.1484375\n",
+ "db10 15.634806632995605 16672.583984375\n",
+ "db11 15.69124698638916 16721.88671875\n",
+ "db12 15.76386833190918 16791.78125\n",
+ "db13 15.807873725891113 16828.6328125\n",
+ "db14 15.84904956817627 16859.560546875\n",
+ "db15 15.879130363464355 16884.310546875\n",
+ "db16 15.916594505310059 16917.77734375\n",
+ "db17 15.97330093383789 16964.7890625\n",
+ "db18 16.010889053344727 17004.966796875\n",
+ "db19 16.06007957458496 17043.080078125\n",
+ "db20 16.109506607055664 17080.361328125\n",
+ "db21 16.15558433532715 17122.78125\n",
+ "db22 16.195322036743164 17152.8046875\n",
+ "db23 16.23825454711914 17190.244140625\n",
+ "db24 16.28815269470215 17229.99609375\n",
+ "db25 16.29660415649414 17237.244140625\n",
+ "db26 16.331958770751953 17263.62890625\n",
+ "db27 16.375545501708984 17302.498046875\n",
+ "db28 16.413320541381836 17331.599609375\n",
+ "db29 16.437959671020508 17352.27734375\n",
+ "db30 16.50661849975586 17411.228515625\n",
+ "db31 16.53733253479004 17433.791015625\n",
+ "db32 16.5701904296875 17458.037109375\n",
+ "db33 16.599777221679688 17484.1953125\n",
+ "db34 16.628063201904297 17505.951171875\n",
+ "db35 16.64190101623535 17514.66796875\n",
+ "db36 16.680456161499023 17541.33203125\n",
+ "db37 16.730104446411133 17587.6328125\n",
+ "db38 16.75263214111328 17598.830078125\n",
+ "dmey 15.428367614746094 16479.267578125\n",
+ "haar 15.07145881652832 16107.921875\n",
+ "rbio1.1 15.07145881652832 16107.921875\n",
+ "rbio1.3 15.1613130569458 16189.78125\n",
+ "rbio1.5 15.32840633392334 16338.216796875\n",
+ "rbio2.2 16.183170318603516 16984.47265625\n",
+ "rbio2.4 15.833732604980469 16793.46875\n",
+ "rbio2.6 15.841042518615723 16820.513671875\n",
+ "rbio2.8 15.870935440063477 16858.255859375\n",
+ "rbio3.1 112.47295379638672 58394.6875\n",
+ "rbio3.3 19.817306518554688 20010.50390625\n",
+ "rbio3.5 18.20765495300293 18729.5625\n",
+ "rbio3.7 17.90874671936035 18495.369140625\n",
+ "rbio3.9 17.855627059936523 18441.083984375\n",
+ "rbio4.4 15.365104675292969 16364.685546875\n",
+ "rbio5.5 15.447882652282715 16390.2890625\n",
+ "rbio6.8 15.320694923400879 16363.552734375\n",
+ "sym2 15.11799430847168 16146.3642578125\n",
+ "sym3 15.206748008728027 16251.126953125\n",
+ "sym4 15.159475326538086 16207.3779296875\n",
+ "sym5 15.204032897949219 16260.142578125\n",
+ "sym6 15.191091537475586 16246.744140625\n",
+ "sym7 15.236370086669922 16293.701171875\n",
+ "sym8 15.241791725158691 16298.5556640625\n",
+ "sym9 15.26644229888916 16327.4296875\n",
+ "sym10 15.264242172241211 16323.5576171875\n",
+ "sym11 15.332569122314453 16396.794921875\n",
+ "sym12 15.310770034790039 16371.40234375\n",
+ "sym13 15.304075241088867 16360.4248046875\n",
+ "sym14 15.317804336547852 16377.61328125\n",
+ "sym15 15.378673553466797 16436.265625\n",
+ "sym16 15.33505916595459 16392.92578125\n",
+ "sym17 15.344695091247559 16404.30859375\n",
+ "sym18 15.363746643066406 16418.08984375\n",
+ "sym19 15.430442810058594 16479.15625\n",
+ "sym20 15.377063751220703 16438.19140625\n",
+ "min: tensor(15.0715) bior1.1 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "#wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " for v in weights[\"17000\"].values():\n",
+ " flat = v.flatten()\n",
+ " #print(flat.shape)\n",
+ " lens.append(len(flat))\n",
+ " to_cat.append(flat)\n",
+ " flat = torch.cat(to_cat, dim=0)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 157,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " conc2.abs(), round(0.1*len(conc2)), dim=0, sorted=False\n",
+ " )\n",
+ "top10_og = torch.zeros(len(conc2))\n",
+ "top10_og[topk_og.indices] = conc2[topk_og.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 158,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(15.5541)"
+ ]
+ },
+ "execution_count": 158,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(top10_og - conc2, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 159,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "42.212055"
+ ]
+ },
+ "execution_count": 159,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(np.sum(errs))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Problem: weights with only a few parameters cannot be represented with the wavelets"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 126,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat5000 = []\n",
+ "for v in weights[\"17000\"].values():\n",
+ " flat5000.append(v.flatten())\n",
+ "conc5000 = torch.cat(flat5000)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 127,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(44.8886)"
+ ]
+ },
+ "execution_count": 127,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(conc5000, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 149,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "dmey tensor(15.4284)\n"
+ ]
+ }
+ ],
+ "source": [
+ "wavelet = 'dmey'\n",
+ "coeff = pywt.wavedec(conc5000.numpy(), wavelet, level = None)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 150,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACeNUlEQVR4nOydd5xcZb3/38/0ur2mbcmmN0pCL6EXQVBERBS8iljwil7vFQsXC6iIBX8oeEUuglcUkA4iSAudhCSEFFI22ewmm2yvU3b6+f3xnDNtZ2ZnWzblvF+vfe3OzJmZZ2Znzvf5ts9XKIqCjo6Ojs6Ri2GqF6Cjo6OjM7XohkBHR0fnCEc3BDo6OjpHOLoh0NHR0TnC0Q2Bjo6OzhGObgh0dHR0jnB0Q6Cjc4gjhFgphGid6nXoHLrohkBnShFCNAshhoQQXiFEhxDifiGEa6rXlQshRK0QQhFCmCbo8WxCiH4hxJkZbrtDCPHoRDyPjk42dEOgczBwsaIoLuAYYDlw02juLCSHzGc53YAoihIAHgauTjvOCFwJPHDgVqdzJHLIfHl0Dn8URdkH/BNYLIQoFkI8K4ToEkL0qX/P0I4VQqwSQvxECPEW4AfqhRD/JoTYKoTwCCGahBBfSjp+pRCiVQjxbSFEpxCiTQhxqRDiQiHEDiFErxDie0nHG4QQ3xFC7BJC9AghHhFClKg3v67+7lc9mRPV+3xeff4+IcQLQoiapMdThBDXCyEagcYML/8B4DIhhCPpuvOQ39F/5npt6ajP1ZB0+X4hxK1Jly8SQmxQvZC3hRBLk267UQixT32e7UKIs7I9j87hg24IdA4ahBAzgQuB95GfzT8BNcAsYAj4XdpdPgtcB7iBFqATuAgoAP4NuEMIcUzS8VWADZgO3Az8EfgMcCxwKvDfQog69dh/By4FTgemAX3AXeptp6m/ixRFcSmK8o4Q4hLge8DHgXLgDeBvaeu9FDgeWJj+2hVFeRtoU++f/Pr+qihKJI/XlhdCiKOB+4AvAaXAH4CnhRBWIcQ84GvACkVR3EhD1Dza59A5BFEURf/Rf6bsB3mi8QL9yJP53YA9w3FHAX1Jl1cBPx7hsZ8EblD/Xok0Jkb1shtQgOOTjl8HXKr+vRU4K+m2aiAMmIBa9b6mpNv/CXwh6bIB6anUqJcV4MwR1nsT8C/17wL1/kfn+dpak25TgIaky/cDt6p//x64Je2xtiMNXgPS4JwNmKf6s6H/HLgf3SPQORi4VFGUIkVRahRF+aqiKENCCIcQ4g9CiBYhxCAyHFOkxs019iY/iBDiAiHEu2qYpx/pXZQlHdKjKEpU/XtI/d2RdPsQoCWqa4An1PBJP9IwRIHKLK+hBvh/Scf3AgLpfWRcbwb+DzhDCDEN+ASwS1GU9/N8bflSA3xLW6f6WDOBaYqi7AS+AfwQ6BRCPKSuRecwRzcEOgcr3wLmIXfsBSTCMSLpmLh0rhDCCjwG/BKoVBSlCHgu7fjRsBe4QDVQ2o9NkXmMTJK9e4EvpR1vV2TIZ9h6M6EoSgsypPQZZFjogTG+Nj+QnGuoSlvnT9LW6VAU5W/qGv6qKMopSIOhAD/PtWadwwPdEOgcrLiRO/R+NUn7gxGOtwBWoAuICCEuAM4dx/P/D/ATLeErhChX8wCozxED6tOO/64QYpF6fKEQ4vIxPO8DyDj9ycCD6nWjfW0bgE8LIYxCiPORYR+NPwJfFkIcr1ZbOYUQHxFCuIUQ84QQZ6qGJ4B8/2NjeA06hxi6IdA5WPkNYAe6gXeB53MdrCiKB/g68Agysftp4OlxPP//U+//LyGER13D8epz+YGfAG+p4ZUTFEV5Arl7fkgNZW0GLhjD8z4GlAAvK4rSNsbXdgNwMTLvchUyn4D6WGuBLyIT733ATuBz6s1W4Dbke94OVADfHcNr0DnEEIqiD6bR0dHROZLRPQIdHR2dIxzdEOjo6Ogc4eiGQEdHR+cIRzcEOjo6Okc4E6KeeKApKytTamtrp3oZOjo6OocU69at61YUpTz9+kPSENTW1rJ27dqpXoaOjo7OIYUQoiXT9XpoSEdHR+cIRzcEOjo6Okc4uiHQ0dHROcLRDYGOjo7OEY5uCHR0dHSOcHRDoKOjo3OEoxsCHR0dnSOcCTEEQojz1UHXO4UQ38lwu1UI8bB6+2ohRK16/TlCiHVCiE3q7zMnYj06OjqTTNd22P36VK9CZ4IYtyFQRwfehdReXwhcKYRIH879BeS82QbgDhJTj7qBixVFWQJcgxzVp6Ojc7Cz6jZ4+t+nehU6E8REeATHATsVRWlSFCUEPARcknbMJahj94BHgbOEEEJRlPcVRdmvXr8FsKvTkXR0dA5m/D0Q9Ez1KnQmiIkwBNNJHcrdSurA7pRjFEWJAANAadoxlwHrFUUJZnoSIcR1Qoi1Qoi1XV1dE7BsHR2dMRPoh5B/qlehM0EcFMlidc7rz4EvZTtGUZR7FEVZrijK8vLyYZpJOjo6B5KhPogMQSw61SvRmQAmwhDsA2YmXZ6hXpfxGCGECSgEetTLM4AngKsVRdk1Aes5vAn54M+XQue2qV6JzpHMUL/8Hda9gsOBiTAE7wFzhBB1QggL8CmGD9Z+GpkMBvgE8IqiKIoQogj4B/AdRVHemoC1HP50N0LTq7B39VSvROdIJRqB4KD8Ww8PHTDe2tnNT5/bylBo4r2wcRsCNeb/NeAFYCvwiKIoW4QQPxZCfFQ97H+BUiHETuA/AK3E9GtAA3CzEGKD+lMx3jUd1vi65e+Qd2rXoXPkEhhI/K1/Dg8Ya3b38sc3mjAbxYQ/9oTMI1AU5TngubTrbk76OwBcnuF+twK3TsQajhj8qiEI6l9AnSki0J/4Ww8NHTC6vEFKnRZMxolP7R4UyWKdURD3CPTSPZ0pYqgv8XfIN3XrOMLo8gQpc01Odb1uCA41NI9A/wLqTBW6IZgSujxByt26IdAB8Kk9FHpoSGeq0CqGQDcEB5AuT5By3SPQAcDXI3/rSTqdqSLZI9BzBAcERVHo9k6eR3BIDq8/ookni/UcgcZLH3awbk8fTosRh8WE05r4PafCzcwSx1Qv8fAiJTSkb0gOBJ5ghGAkphsCHRUtNKR/AeP84Okt7OsfGnb9iYYtrLC3882bfoUQE19yd8QS6AdhBCWq9xEcILo8UnlnspLFuiE41IiHhvTYLIA3GGFf/xD/ee5crjttNv5QBH8oij8UwfKXX1I+sJmOgZ9QVWSf6qUePgz1gbsKBvfpn8MDhGYI9GSxDkSCibJRPVkMwM5O+T7MqXRjMRkocliYVmSnocjADO8m7CJEY1PjFK/yMGOoHxwlYLJDWDcEBwLdEOgk0HoIzA49NKSyo0MaxjkVrtQbWt7GEAsD0Nms6zJNKEN9YC8Gi0P3CA4QcUOgVw3pxPMDxbXSECjKlC7nYGBnpxeLyUBNqTP1hl2vxv/0tu04wKs6zBnqA1sRWJx6juAA0e0NYjYKCu3mSXl83RAcSmgVQ8W1oMT00j2kRzC73IXRkJYMbloFNacQxQB9u6dkbYcywUiUV7d1cvNTm3ltR9r8j0C/9AjMTt0zPUB0eYKUOq0Y0j/nE4SeLD6U0BLFxbXyd8gnd2VHMI0dXo6tKU690tMBnVvg7B/i7dhNsW8fA0PhSdtNHS4MBsK8uq2Tf33YwaptnfhUlcu2gQCnz1VngChKUmjIqW9GDhBdk9hDALohOLRIDg2B7CVw5SHW2tcCsQiUzp60pU0FPrVi6MrjZqbe0LRK/q4/g+jmF5nlb2Nr2yAn1KcPxdMBiMYUvvbX9by0tYNwVKHMZeGjR03j3IVV/M9ru+j2Jg0NDPshGgJ7kZ4jOIB0eYJUFtgm7fF1Q3Ao4e8GgwkKpsnL+brl/7wRfJ3wxVcmb21TgFYx1FDhTr2h6VVwlELVUmyVDdS2beCx/UeOIdjV5eXVbZ1ce2p9Xsfv7vbyz83tXLS0ms+dVMvRs4rjobanNuxj/Z7+xMGavIS9GCwu8PcNezydiafLE2TxtMJJe3w9R3Ao4esGRxlY1RNfviWkvk450OYwSy5rFUNzK5MqhhRFJorrTgeDAUflHIqEj6a9rVO0ygPP4+tbufUfW+OVJiOxvV1+jr58+myW15ak5FtKXdZUj0DrKrYX69VrB4hYTKHHF5rU0JBuCA4l/D3EnKX8X/vbDBhE3l/CplA/25WhVGmAw4DGTi8Wo4FZyRISXdvB2w71K+Xl4joABvcdOZVD/X5ZNru9PT8Zku0dHgwCGtJLcJGdrFqDHpCYRWArkqGhIy1H4OuBvWsO6FP2+UNEYzJkN1nohuBQwtfFVoeb2xv/xquO/HdjvzAHuKmsFPpbJnmBB5bGDg/15c7UQR1Natno7DPk7xJpCER/M4HwkTFoXTME29oH8zp+R7uH2lInNrNx2G2l6smnxxuSVyR7BBbXkVc++vad8KcLDuimqsurNZNNXo5ANwSHEr5umq3yw+AziLxDQ4PE2G02EettmszVHXB2dHiZW5mWH9j1KpTMhqJZ8rKaWJ+ptNPYcWSEMfqH5El7W54ewY5OD3Mqh3sDkGhg0k5GCUNQlAgNHWYhx5wM7JWFF0l9KpPNZHcVg24IDi38PTSru1+fMOTnEUTDeAUEDQY6uj6c5AUeOLSKoZSO4kgImt9MeAMAFicRRyU1ooMt+weGP9BhSJ8v/9BQIByludvHvHSDqqKJnCU8gn75WysfRYFIYLxLPnQYbJO/d750wJ5SNwQ6CSJBCA7SjIzV5u0RBD3yWGB33+ETJ0/WGIrT+p7UvtHyAyrG0nrqjJ1s2Z9fqORQZ2BIGoIdHR6isdy79V1dXmIKzK3KbAi00FB3skdgMMmwkNbDciSVkHqSDEEsdkCesturGwIdDVVnqCUmY7I+kyW/ucWBAfxC/ptbPIdP5UxcYyg5pNH0KggD1J6acqwoqaPe2MWHbUeGIejzhyhymAlGYrT05D5Ja+9jNo8gkSNQDUGgXyaKhZgwQ/BuUw/9/tC4HuOAoCjSELgqwdsBHZsOyNN2eYLYzAacluE5nIlCNwSHCv5uFKA51A+Az2TO6wuoBAbiHkFLsGcSF3hg2alWDNUkVww1rYLpx8r4dTIldZTGumlq6x5xhzzZRKIxVjdN3v8hGIniD0U5vq4EGDk8tKPDi9koqC3L3KFuNRlx20x0JyeL7Wont1l978dhCALhKJ+5dzV/fOMQyF8F+mUYbNmV8nLjvybneV7+MTz8mfhFbVbxZM7U0A3BoYKvmw6jkaGY/EL6jKa8QkNDQz3E1A9Qc2wIYodH5cyO9IqhoX7Ytw7qzxh+cIlsrCoLt9E8wg55snl2YxtX3PMu61p6J+XxB9SKoRW1JRjEyAnjHe0e6stcmI3ZTwXlyb0EqiHwhrx0o4ZG8iwhjWb47HV7g0RiCtvaDoGJe1p+oHopVB8FjZOUJ9i7Brb9I56Y7/JO3qxiDd0QHCr4umkxy0ZwgzDgMxjzShb7NelqoNlkksNEDgMaO72p+YHmN6QQX1p+AIj3EsiE8dSGhzbs7Qfgpa2dk/L4faohqCq0UVvqHNEj2N7hyZof0Ch1WZIMQT/Yi/j5ez/ny9v+V16Xx+ew1dPKigdXsKVnS8r1WiK0sfMQqOjS8gPuaphzDrSumZwyUn+v/Czvfh1IeASTiW4IDhX83TSbpWja7KLZ+AyGvOYW+wIyDDHLUsR+k5FQz6E/pMUXjNDal1YxtOtVqYY5Y8XwO6i9BHXGTj6cYkOgPf+r2ybHEGix9mKHhXlV7uG9BOFEhY/2Ps7LUjqqUeaypvYR2IvZ0rOFzpBahZVHL8Eezx7CsTDvd7yfcr1mCPb2+RkKHeTeqqdd/nZXw5xz5cl61yTItmgqw+pj64ZAJ4Gvm2aLBbvJRl1BHT6RX2exd0iGIBYWzUURgj0dH0z2SiedXV3ydc9NTxTXngKmDN2X9mKwFbLM0TulJaSxmMKW/QM4LEa2tXsyzlkeL5pHUGg3M6/KTUuvP9EV3PI2/Gw69OwCErvwOVkSxRplKaGhfiLWApoHmvFEhlAgrxyBV/2s7uzfmXK9lntQlMT/9aDFs1/+dlepuajiiQ8PKQr41RzSzlcIR6L0+cOTNqtYQzcEhwq+LpqtdmoKanFZXPiFkleOwK9KAiyadjwALb3bJ3OVB4QdHWknsL4W6G1K7R9IRggormOOuYsP9w+iTFEDVEuvH18oymdOqAHglUnwCgbUZrJip4X5VW4UhUQj3fsPymYodT7DjvbcFUMapS4Lff4w4XAYggPsNZsJx8JElAhBIfIaV+lTj2kaSE0KJ+shaRVMBy2edlVjyQ4GI8w+c+LLSIOD8n9UUg8De+hvld9X3SPQkfh7aDabqC2oxWl24lViee3EfKr7vrDiKACaB/dM5ioPCI0dntSKobjs9MrsdyqpozraRo8vRMdgfmJsE43mjXx02TRqSh2TEh7SPIIiu5l5VQWAWjkUCcLWZ+RBAbmO7R0ebGYDM5MrrzKg7Ub7e6UM+i4Rid/mFSKvz6FHLXXe2b8zxRB3eQO4bSbMRnHw5wkG22RYSGPOuVLQsX0CvWzNG1hyOQChHS8DkzeiUkM3BIcIIV8X+w0KNQU1OM1O/ESJ5REa8ql5hApHBWUYaQ50j3CPg5/GTm9qxVDTq/ILWj4/+52K63AF2jAS5cO2qQkPbd43iNkomFPp4ox5Fby1s3vC4+L9/jAWowGHxcisEgc2s0FWDu16BYLq61YNwY4OD3Mq3MOnu6WhiZ3FDUEsEdLyGgx55Qi8YflZ9YQ8dA8lPoNdniDVhTbqypw0pnsEmx+DptdGfOwDhifNEMw+S/6eyPCQX60mm74cimqwtEgpC90j0AFgT0AW69UW1uI0yZrvobBvRJ0Xn/oFdJqd1JhctEQPki7Qxpfgj2dCNDzqu+7o8KQqZbauhVknyhBQNkrqMSgRpolutuybmoTxlv0DzKlwYzUZOXN+BcFIjHeaJtYw9/tDFDrMCCEwGgRzK91s7xiUJ1Wr9BAIyNe/oyO7xlAymkfg7VMNQThhSGVjYx65qnDimOQ8gZYInVPhTvUINj0Kj34e3vrNiI99wPC00a4U893HN0mvxlUO046GnS9O3HNoVX6OUph9JkUd72IiohsCHUlLWH556wrqcKiNPD7BiDXcvoi83Wl2UmuvpNlI/nMMJpPGf8m6f60SI0/8IVnpEhebC3qlEFjFgtx3VCuHjiscmJISUkVR2LJ/kEXT5Mn4+PoSHBbjhOcJ+v1hih2JkZzzKt3s3t8N2/8Jiz4m5SECA/T7ZYhspPwAyJkEAP5B1RAEu3GZpQHxWOx59RF4Q16sRvk4yXmCbm+IMpeVOZUu9vSqlUMt78CTX5EHBA4SfahYFLwdvN9v529r9tDap3pFc86V0ib+CeoL0UJDjhKYfSbmiI+jxE49WawDRILsRu6cawpq4l9Cbx56Q77IEAKwm+zUFtTQZzQy0Lkl530OCN1q0to3uhNhXGNI8wi0ctiyubnvqPYSLHcPsGUKQkPtgwF6fSEWT5dTpqwmI6c0lPHK1s4JTV73+UMU2ROVU/OrC1gaeE/u2hdfJr2CwEA84T5SDwEkQkPBwV4iwG5/O0vLlwLgtdjyqxoKe5nhmkGBpSDuESiKIj0Cl/QIFAX2Nn4AD10p1WPrzzh4DIG3E5QYmwflJmzNbvXE33DOxJaRaobAWQZ1pxHDwFnWDzNKhE8kuiE4FFATxWUmBy6LC6dZhob8eSiQ+qJBnBgQQlBbthCAlo71k77kEelWT+C+0YVGGtMrhrpUIb3yebnv6K4Go5X51m729g7FhdkOFFo4SvMIAM6cX8H+gQDbJ7BaZmAoTFGSRzC/ys3FxrcJ2cpkea2tEIKDSdPdRjYELqsJq8lA2NfLXrOJcCzCsvJlAHhN1rzLR10WFw1FDTT1S4/AF4oyFI5S7rYyt9JFCYNM+8fVIIxw1d+lhPjBYgjUZrIdfvnde69Zi+UfA/aSiVMj9feA0SJF/exFNNvmc5ph8jWNdENwKODrosVsptZeCZAIDeUxpcwXDeEQsiO5pno5AM092yZxsXkQ9CQ6nL2j8wh2dHqkNk6pWunSvV2eOEpm576jwQAldcxAhqK2HmABus37BxACFlQnDMEZ8yuAiS0j7fOHKHYkeQQlgjMNG2gsO0uWPNoKVY/Ag8tqYlrhyMNOhBCUuazE/L00qU2NCUNgydsjcJld1BfVxyuHkuWVawoM/K/lV1iHOuHKh2T5pK0wns+YctQQZrtSQpnLyhrNEBiM0HAWNL44MWWk/h6ZH1DzXeuMR7Mg1jjpg3B0QzAWIiF46YcHbrfi66bZbKLGNR0gHhqS3cUjGAIljNMgv7wzShdiVBSaB6d4Ull3ors56ukY1V0bO7zUl7kSFUNd22X8P1MjWTrFdRQHpALrgc4TbNk/SF2ZE6fVFL+ussDG4ukFvDKBchP9/lSPoLT1FewixCqTqsiqGoLt7R7mVrryFjIrc1kQgQF22uWOeEn5EgC8RnN+OYJwwiMYDA3SE+hJGAKXGcvTX2KZYSf3ln8HZqrd4bYCiAZTuqGnDLWZrFsU85kTZtHU5Uv0QDScI7uB2zaM/3n8vdIQqLweW4yBGOx+Y/yPnQPdEIyFfevgzTvkLuAA0D+4l36jkboiuevVQkMjegSxGD4liiFq4r43d2M2WZihGGkZmhx5g7zpTsxF6O/aP6q7NqZP0+reAWUjhIU0SuowDeyh3GU54B3GW/YNsHha4bDrz5xfyfo9ffT5xi/DPBSKEozEKEryCNjyOL3GMv7lkU1s2ApQVI9gXh75AY1SlxVTsJ9dVjvTnNMosBRgN9nxGvPTvPKGVI+gUAoA7urfFe9WXrj5V7D1GR4v+zIP+45O3Mmmvl8HQ3jI004MA2WVMzh1TjkAazWvoOEsQExMeMjfLRPFKm/6awgaHJMjZZGEbgjGgpbQ8XUdkKdrVmOqtSWyTj5RNTSC3lDIi89gIBQ08rtXpTteY3LSHJnaqiGlawcRjLQqZfh62/K+nz8UYW/vEHMq1BNYNCw7istHSBRrlNRD2MdJVdEDqjnU5wuxfyCQkh/QOHN+BTEFXtsx/s+SNqIy7hEM9UPji2wvPYftnT4pwW0rJDY0QJ8/nHgf86DMZcESHmSX2cjspA3JaPoIXGbpEYAsIe3yBCnES+nGP8DRn2Hv3H9jT68/MVvaViR/B6c+PKQM7qeLQpbOKmXJ9EJsZkMiPOQsk7mCiZCl9veAowyQhr0vCPuKV8Culyd1JOiEGAIhxPlCiO1CiJ1CiO9kuN0qhHhYvX21EKI26bbvqtdvF0KcNxHrmXQ0USjv6MIaY6XZsxeAGjXZm+oR5IjPBgfxGQSRsIleX4jmHj81tnL2iBixKZSj9u3/kOZYJR1KMRFP/t7Jrk75WuMaQ71Nsh0/X49ArRw6rqCfnZ3eAzbMXgtDLcrgESydXkiZyzIheQJtRGW8fHTbPyAWxjfnowTC6pAaW1F8hz1aj8ASHaBZxOInc5fZJSvXRggNRWIRhiJDuCwuyuxluC1umvqb6PIEOdYodY9YegVzqtzEkjWH4n0PU+8R+Hv20RYrZtnMIiwmA0fPLE4kjEGGh1rXjr+MVMsRkJhM1l99CvTvkZ/3SWLchkAIYQTuAi4AFgJXCiEWph32BaBPUZQG4A7g5+p9FwKfAhYB5wN3q493cKN5BN4D5BEEOjEpCtMLZgJgM9owCqPMEeRyy4Me/MJANCprkNe19FFbUEPAIOjsmroS0kjHdnYq0/EaizAN5V81NGwqWZdagpq3RyANwXxrD5GYkqgFn2Q2q2GoTB6BwSBYOa+C13Z0EYmOL9moKY8WauWjmx+DohrK550IqFIT1gKMET8mInlVDGmUuawEzD5CAuqLZHjHbXHjRRm5YEHVGXKZZU6ioagh7hGcaG2SU+WmHR1fj1YifDCFhiL9++hUijl6ZhEAK+pK+HD/IJ6AWn1WcyKgQPs4KnyiEenFqYagU81BhGpXytubXh37Y4/ARHgExwE7FUVpUhQlBDwEXJJ2zCXAA+rfjwJnCZmlugR4SFGUoKIou4Gd6uMd3GhWf5Q18GOlJdjPDMWIWU36CiFwmB1qaCjHlzAwiNcgiMVkZci6lj5qS2V4aXfb2klfd0aiYdy+FjosM3GUVOMM9xLLc2pYY6ecplVTqk7T0noRRuoh0CicCcJAZVSGozoGD0wScsv+QaYX2Sl2Zk5onzm/goGhMOv39I/refrVkthipxl8PVKDafHHmVtVgNCG1Kgn15mOaLw/IB/KXBZ6LfLEpHkETrMTD7ERQ0OazpDLIg14fWE9uwZ20ekNcIxhJ1QsBKub2lInJoNIiM/ZDh6PwOzvoMdQSn25fA3H15UQU+R3Coh7m/Q1j/1JAv2AEjcEWjLaVTVX9lXsOrgNwXRgb9LlVvW6jMcoihIBBoDSPO8LgBDiOiHEWiHE2q6uid2J9/lCI852TSHuERyg0FDMT60htbPQaXaq4ypz5AiCg/gMBiIxO3MrXaxv6aOm6hgAWnq2TuaSsxLracJIFFPlfOzF1RTjobUnv5xFY4eHujJnYppW1w4omA7WPHe2JgsUzqRIrRxqGzhQhmAgozegceqcMkwGMe7wUJ/qERTZLbD1KVCisOjj2C3GxJAa1RAsKVVGNfqwzGmh3SJDaVrC121x4yMKsbCspMuC5hG4zfL/1FDUwEBwgE5vJ/OjjTBDljVbTAZqy5wJtdSDxSMID+GIDmIorI7rMh09qwiTQSTCQwXTZde2quw6JuLyEjJZ3KUNrS+wyea63a+PSZIlHw6ZZLGiKPcoirJcUZTl5eXlE/rYP3luK9fctyb/O2j/sAMQGorGouwhQq0p9UTiNDnxG805PYKQv4eIECg4uXBJNTs6PVgLFmKPxWgZaJ7klWdm306p1Fg1eylF5dMwCIXG5vzWsqPTk6qd3709f29Ao6QOh1eWz47GI7jn9V1sah39CckXjLC725cxP6Dhtpk5rq5k3Gqk/ZryqMMMmx+H0jlQJcs851W62d7hQVF32QtKRpd4LLcpNFsMlAtHvFjBZXbhUVQl0hxS1JpH4FSH3WuhJUPgfZyKN2WY0JwKV0JzSDMEU5wsDvbJyjZ32cz4dQ6LiUXTCxMdxkaT3LWPxyOIy0uoOQJPECGgxGmRktfBQVmxOAlMhCHYB8xMujxDvS7jMUIIE1AI9OR530ln874B9vcH8m/1T64amkgt8gy0+doICai1pRo/p8WJz2jMmSz2Dcl1Om0lrKgtQVHgg/YgtTHBbv/ovRl/KMIZv1zF85vzr/RJp2PXRgAWLFlORbX817e2jtzX4AmEpcaQVukSi8l+hJE6itMprsPQ10yRw0zbQH45gmAkyk+f28avXhz9LIetbYMoCiyent0jABke2t7hobUvv/m/mej3h7CZDdgCXdD8Jiz+eLwxaV6Vm+YeH/uGZDhoTsHoEuVlJj9NZjOVojh+ndPsxBdTd6g5Poea4FyyRwDgUDbLA5INQaWblh6fTOSbHbJZcIo9gt3NMqFdMb0u5frjaov5YO9AouiguA56x+ERJMtLID2CEodFesB1p8lcyiSFhybCELwHzBFC1AkhLMjk79NpxzwNXKP+/QngFUWedZ8GPqVWFdUBc4BRbM3HTyQao6nLRygawxOMjHwHSPzDYmE1rjd5NPfJ5qsa17SU650mJ94R5hZ7A3K3UuwqY9nMIgwC1rf0UWtw0BIZ/S6rscPL7m4fP39+uyxFHAOhjm10ilKmVZRjKagCoKu9dcT7rW3uQ1Fgea16IhpsldUqY/AIGOql3h2lfSC/uQTtagjpzcbuUdf7b96nJYqzewQAp82Vhn5109irTqTgnEWdO6DAoo/Hb9OG1LzWItdf6xqdIXApg+w2mymmLH6d2+LGFwsRhZx5As0QaDmCcns5LrMLq2UfQZNLei4qcypcxBRo6vJJI6Y2wE0l+/fIap26utTu9ePqSglFY2zUPMXi2vGFhtI8gpQRlY4SqXQ6Sf0E4zYEasz/a8ALwFbgEUVRtgghfiyE+Kh62P8CpUKIncB/AN9R77sFeAT4EHgeuF5RlANa19jc4yekVmvE57KOhL8HXFLuYbQSCaOluftDAGoLalKud5qd+A0iZx+Bxy8TWRWF5bisJuZXFbB+Tx81tjL2EyEUHd1JTSvr293t49mNo2sEA4jGFNzeJgac6s7KKU9+nu6RH+vd3T2YjYJjZqmGQGtKG61HUCLDEkvsvXmHhvb3y+MiMYV/fTg6tdQt+wcpdVqoLMitHjmtyA5A7zgay/r8YQrtZlnPXjIbKhLzGbRS0Wcb5c59mnV0z9PWv5OQQWCLVcWvy7exURtTqR0vhGC6s46AdYCBkmVS/kNFqxxq7NQSxlMvM9HfIYc5lVbXply/vEZ+FtfsVk/gJXXSaI1VDkIzBHY1R+AJpqqOzj4T9q2VlUUTzITkCBRFeU5RlLmKosxWFOUn6nU3K4rytPp3QFGUyxVFaVAU5ThFUZqS7vsT9X7zFEX550SsZzQkj8fr8eaxQwwH5Idekz2e5IRxc/9O3NEYpaoh8Ici3PDQ+ygxq5ShzvEF7PbK3eXMIvmBPbammPf39DPLPYOYEOzt25n1vplobWvjJet/cnXxZn77ys68q300tuzrp1bZj1Er93RJQ2AK9Iz43q9u6mXZjCLsFrW6WBOby7eHQEOt7phr7so7WbxfnS3sspp4duPowmJb9g+yaHrhiIlZp8WIySDiTWFjYWAoRLkdGRZqOCvltppSJzazgc09ch2O2OiaCnf2S880Fk4MZnFb5EnbJww5ewnioSFLIr9TYa6m3RIlVHlMyrG1ZQ6MBpGUMC6Yco8g1NdKSFgSDW4qxU4LcytdrGnWKodq5e+xhof8PVJsziyr/IYNrT/q0/DZJ0DNtUwkh0yyeLJINgTd+XgE6jB4ylVDMMndxc2De6gNhxHqSfP1Hd08tWE/Hf3IGu4cyeJen/yA1pVKV/OYmiK8wQguizwZNrePToW0tPk5GsR+vmV5nJ2dHv65eXS74w+2bsMthiivlxLG2IqIGcyUiQE+zCEC5w1G2LRvgOPrE633dG+X82OdZVnvlxH1y1pr6KDHFyQUGTnHo+USrlgxk7d3jWy0NIKRKDs6PDkrhjSEEBQ5zPFRk2Ohzx/mWLFdnpRnpxoCo0Ewp8KNFxsxxKgTsE2qPpV3KBGijM8kMBhy5whCXswGc3weAUBlyEif0cjgtNQ5ElaTkZpSR6pHMIXJ4h5vEHuwiyFbRcbBR8fVlbC+pU/2gIy3hNTfE68YUhSFbm+aISipl+NYjebM9x8HuiHo8OBUd5l5ueWa+xb3CCY5NORvpyYSibeda1IE/V6BnyhKji/goGokGsrkfY+dJT9kA2HVEIyyqeyo3ueIYKRwYBuXFTfx21caR+UVaBVD7ulqv6EQ4CijjNzDYta19BGNKRxflxDjokvVGBpFCSQAVhe4KqmKtqMo0OkZ2SvY1x+g1GnhsmNmEI0pPL8lyQD274H+vRnv19jhJRJTMmoMZaLQbmZgHIag3x/m6PB6MJil5HQa86vcKBgIGZ2j3mXv9LdRHYmwbyhh1BLih7k73DV5iWRmDsrn7ywdbsjnVrgTHoF1aj2CD1r7qRK9iOQRlUmsqC3BG4ywtc2T8AjGmidIkpfwBCMEI7FJn1WsccQbgu3tHlbUyRNkXjs9rXS0dLb8wk1iaMgf9tMRHqQ2HAZnGYqi8LpqCLoGIQoEc4SG/GpJX5VbbSIqsVPmsrKhr4zSSJSWpElRIxHt3sWi6Dbenv55cJbzX+4X2dbu4aWt+b3+UCRGpF2Vv06K6xvcFUy3+HJq/7zb1IPJIDi2JlGxQvf2/DuK0ymuoyQkE9T55AnaBoaYVmRnQbWb+nInz36QFB568quJaVppJBLFI3sEAEUOy5hDQ4qi0O8PMd/3Hsw6QRq8NLQ8QdQy+pNrU6Cb2aEIe7yJU4aW/B3JI/CEPPFjNearxrMtOjw5PqfSRXOPj2AkmiKJMRVs2DtAJX04SmdkvP049dyxprlXvufO8vGFhtKaySZ7RKXGEW0IgpEozT1+Fk8rxG0z0TMaj8BZDq6KSQ0N7fHIJFVtWH4hdnX52Nc/RG2pgwG/9GJ8EX9WMaqhqAxpJCfpjq0pYtV+QW0kSos//9COZ/WfiSmCgfmfghVfpKrzdU4r7uHOVxrzKrvd2NrPzFgrYZMrkWgHcJYzzeTJqQa6uqmHJTMKExLOvh75fxhtfkCjpA6XTz0R5ZEn2N8/RHWhDSEEFy2dxurdPQlPYmAvdH6Y8X5b9g/ispqYVeLIa1nFDnO8F2C0+EJRimN9VPobZVIxA8fXlWIxGjA5RleJE41F2R0epCYq8IVicpwkJE3Ky50j8IV9qR6BorBoYCu2mGDXwK5hx8+pdCcqh6Y4WbxhTx9Vhj5MRRn7XKkutDOj2M57Wj9Bcd04Q0OphmCyR1RqHNGGoKlLKjLOrXJT5rLGRZ5yoslLOEqlMZhEj6BZbfqqNbnAYIiHhb6ycjaKqh+Ua25xCGnYHKbEiejYmmKaewPMMthpDuf5BYvFsH74KG/GFlM9sx5WfAFMNn5Q9hqb9w2yavvIxvCdXT00GPYjytPCOc4KShigqduHPzS8fNcfirCxdYAT6pPCQpq0xGgrhjSK6zD52rESipeG5qKtPxCv6rloaTUxBZ7X8iPeTvkFziA2tmX/AAunFWAw5Be+KrRbxmwI+nwhTtEmWWUxBEtmFLLlx+dhdZWM6uS6z7uPIDFqFJnE1L4n2i7fO4Lm1TCPYGAvhdFeiqJF7OrPYAjUMaSNnV6ZLA55pA7PAUZRFJr27sdGCNxVWY87rq6E95p75YaouHYchqBX9wimgsS4PhelTkueOYJuQEiX1VUxqTmC5sFmAGZZpfv5+o4u6sudnLeoCmKqIcgyt3gwECYkIlgVA0ZDQsdPC6+Ui0J6iTAQzGNnuOcd7L5WHoueyuxyl0zQLvsU9fufYVFhiP/38shewdu7ephnbMNUkXbydpXjisgv0bb24aWw61r6iMQUjq9LShRrYnOj7SHQKKlDoFBv7hnREAwGwniCEaYVyZPg3Eo3cytdMjwU9CaMcE/qCS0aU9jall+iWKPIYY4Lx42WgaEwpxk3ErKWQNXSrMeZjYZR1+ZrM4ZrjDK0FDcEyR7BCH0EKR5B63sA2A2zUgbZa9SXOzEIKSkyld3FzT1+bEH1+50lRwBwXG0JPb4Qu7p8soR0oDWn5EZGtGpETV5CNwQHjh0dHkwGQX2ZixKnJb8+An8P2ItkS3laaMgf9vPg1gfjO/nx0jzYTLVixO4sJxCO8m5TD6fPLafIYaHMpZbuZdmN7e70EjFEsQtTyvWLphViMRqwReXOY08+08o++CtBg533bCclhNNOuB4RDfKzmavZsLefN3dmVxENhKNs37OPMqV3eFzfWY4hFqYAf8Y8weqmXowGwfLa5IqhHbLrtHDmsOPzomgWAEscA7SPkCNoU3sIqgvt8esuWjqN91p6UxvhelJLcXd3exkKR/NOFAMU2c34QtG8KpnS6fMFONWwCc/0U1Pq8jMySkOg7drrLFouTX5P7CY7RmHEa849t3hYaKh1LQEsOOyL6A300htI9aasJqmN1NjhnVJDsGFvH1VCLQ3NYQi0HON7zb1q5ZAiiwhGQ7q8hDeIySAosk98hVAmjnBD4KW2zInFZKDUZaXHl09oKJHZxyk9AiUa5bmm57j4yYu5bc1tfOu1bxGOjb36Q6N5oJmaSAwcZaze3UswEot3oDao1RZ+kbmZp6WzlyEDuIypOwqb2cji6QUMDsk4fXP3CJVDIT9seYq3racyrTwpPFM+F+acx5L9f6e2QPDbl7P3JKzf08esqHrSTN/FO+Xc3lqbP2Pl0LtNPSyeXogracQjXduhtGHkE142VEMw19Y3YrJY6yHQQkMAH1lajaLA6k1JuYGexpT7bdaG1Y8gLZGyLNXIDgyN/rMTbdtEmRgkXHvGyAdbCyAfT1Bl18AuqmNQqJYwax6BEEIOpzFZRtQaSg4NKa3vsTFWz3RXYlpZOnMqXezo9EzpTIINe/qZaVKftyC7Iagvc1Lmssg8wVgrh9LlJdRmsnzDiuPlCDcEHuapnYxlLhkaGrEc0tedmCnqqmCr2cDn/vlZbnzjRkptpVx/1PXs6NvBgx8+OK61KYpCy2ALtaEAOMt4fUcXFpOBE9QSyvmV8kvpzTK3eH97Oz6DAZdp+HDyY2uK2dFTKecXd23OvZBt/4CQh4eCJ8mwUDInfQ3h7+anDdtY276Bxzdl7kt4Z1cPc4xq93B6glf94B9TGh7WSzAUivJBaz8nJIeFQHoEY80PALiqwGih1tg9YrJ4/4BmCBLv4+xyFwuqC9iyQzV+wpDiEQwGwvz5nWYcFuPw9ywH2u5vYAyVQwWtrwFgnnPWCEeSSMDmqZO1q38X9aEwFpfqESSFUN0WN16jKatHoChKqkcQCULbRtbHGphdODv++OnMqXDT0uMnpOoTTUXCeEPrAEsL1ZBXDo9ACMGK2hJW7+6Nz7wYdZ4gXV7CG6TMnb9M+Hg5Yg3BUCjKnl5/fMhJqdNCTEloumdFTej0Bnr5Uc9qrphWRbOnhR+e+EP+9pG/8aWlX2LljJXc/cHd7PeOXoZBoyfQgzfspTbgA0cZr+3o4vi6knhn7dJquZPO1t7f2d2NTwhc5uEVK8fMKqYlUsn0SITmDF/CFD74G7GCmfzL30B9eVpHY+2pULWEiu4Hcdb+gZvXfZ4vPPlzfKFUz+rtXT2c4O6R5bbajknDJV/H4qIQ29oGU4azrN/TRziqpCaKg15ZqTPWiiGQnkThTKbRRedgMKfx398/hNEgqHCnGtSLllbj6VH/v9XL4jmCXl+Iq/64mo2tA9z+iaUJyew80EZMjqWprKLzLbbGZuEuz1zmmIKtEFByS5irRGNRdg/spiE4hMlZgttmSgxtRx1XaTRmzREMRYaIKtGER9C+GRENsiHWQF3xNFxmV1aPIBpT2BdQQyMH2CMIRqJs3T/IXLtX5gPN9pzHr6gtYV//EG3RAjDZR19Cmkln6ABVDMERbAh2dnpRFOIeQan6po/YS+Dv4R2L4KInLuLJnve5atDDM8d+n8vmXobRYEQIwXeP/y4AP1v9s/wVTdPYPSA/SLXhCH2GQnZ2ejl9bkKB9OiZsorBn2VucU9PN36DAacpgyGoKWavUk5NOEKLL4dkwmAbNL1KR92lKBiG726FIHrC9fzA6KHAZKFUHM2agb9w8p8v48F1a+RuMBjhg739LLF1yN4LY2rOQgsNNTj8BCMxdncndparm3owiCShOUiEYMbaQ6BRNIvyaAehaIzeHAnatv4AVQW2uA69xkVLqykXAygImHkC9OyiY8DPFX94h+0dHu65+lguWjoty6NmWZI6WWzUlUNBL9WDH/COOAqLKY+v9CgGvuzz7iMYDTI7FAZ7MWUua4pH4DK78IrsVUNxwTnNI1ATxe/HGqgosFFfVJ+5hFRVmd01aMp7rRPJh/sHCUVjTDf15/QGNBrUSqe9fYGxVQ7lEpw7ABxRhuCJxidY0yZPUNu1iiG1yaZUjc/mlJlQFP6Bj696N1HpqOTRU37Njb39FARS3eJprml8ddlXWdW6ilf2jE0tcF2H1B2fEwqzuU/uik5LMgTTC4sALVmc+vyKouDp78FrEDiT9F00KgtsFBeXMC1ioCXUTzjbsItNj4AS44OS8wGGewTAX4x+NtqsfC/q4rVr/sTn59xMzNjNzzZ+iQvuv4WH3msmElOYEdmbucrHUQoIZljkCSM5T/Du7l4WTy/EbUtKmI1VYyidolkUBOSOPlfl0P6BoZSwkEZNqZO5ziEGRIEMU0WGuP73z7K/f4j7/20FZ86vzPBoIyxJ9QhGXTnU/CZGJcIGy7H4w37u2nAX33j1GwxFsshsxwe+jBxu0Xbrs8NhsBVR5rLQneQRuCy55xYP0xlqfQ+/rZIOSih32ZhdODujR6BVDm3vVw3wAU4Wf7C3H4CSWE/O0lGNEvX80esLyvDQmHIEAuzFxGIKPb6Qbggmg2gsyn2b7+ML//oCVz13Fa/seQmLEWrUZp+4R5AjYfzAxnv4TnkxR9mreOCCB5hdrQpmZegluGrhVcwtnstP1/w0PqEpXxRF4eldT3Nc4RwqolFWdwiqC23x+moAgzBgVKzyS5i2G+sYDGKJ+vAZDBkNAcg8wawhJwFivLHvjUyLgA1/gxnH8cFQGWajYGZaY1TLYAu//eD3rHTM4sLd66B9E9886XL+dfkzNLiOZZ/h7/xq0zdw2Dqwe/dkNgRGEzhKKGYAi8kQzxMEwlE27OlPLRsF2UMgjHEV0TFTNAtrsAcbwdyGoD+QUjGUzHz3EO3RAt4fknmO0kALf7n2eE6aPUr9I21JDi1HMEqPYNfLBISVD0uCXPzkxfzPB//Dy3te5pfv/TLz8aOY/KXt1utVj6DUmdpvIz0CJWtoKF15lNb32O9eDECZ28Lsotn0BnrpC6QqdtrMRmpKnWzuVvJe68t7XuaiJy7i7zv+TjQ2PhHjDXv7qXBbMfs6oGBkz65UHfvZ4wslPILRRAP8PVI7y2Ckzx8iGlP00NBkYDQYefSjj3LziTfTH+znjcFf4Zx9B880PUU4Gk78IzN4BDElxi/f+yW/3PA7zvH5+Z+Gz1BgKZCxQ6Ml4+xis8HMzSfeTJe/i9+9/7tRrfX9zvfZ69nLJYVSk+f1fXD63PJhCpZWowO/wUDQl/olaery4hZ+/ELgtGYuXzy2phi3p4LiGDy765nhB7R9AF1bYdmnaOryMqvEkRLvjikxbn7rZixGC/991m8QZiesug0UhSpXOU984o98b/kt2Bxd2Ot+zwcWY/YEr7MCo6+L+VXueIfx+3v6CUVjqfkBkBVDJfVy7OR4KJJqrtNFd9YS0lhMoX0gkFIxlMw0k4cupZDr/iHX/KOTrRw9qzjjsfngspowGkR85GS+bNr9Ep+aNp0Ox18os5fx5wv+zOcWfY5HdjzCqr2rht9hNIagfxdVliJciiJDQ27L8GQxsazJYs0QuC1uOdGvv4Um60KcFiMOiyk+pCaTVzC73MWuniGwuEdca6unlf9+87/p9Hfy43d+zFXPXcWmrrEPkt/R4WVRtRPh7RidR+ANyRLSsH90PUbJXcWqoS3TPYLJwWq0cvncy3nm0mew930Om8nGzW/fzPmPn8/fGv+IybWNvYPtKXH9cDTMd9/4Lg98+ACfmnEmv+jsxupSPxhCqCWkmTtrl5Uv45PzPslft/2VD3syyxBk4qldT2E32TnbKuPne4KOlLCQhsviYlAY6elNrcNu6vbhxE/AYMBpy3xiOmZWMS3KNC70eFjV8hKDL/8QOrclDvjgIWnkFn2MXV2+YfmBv237G+s71/PtFd+momQOnP5fsO1Z2PIEICsprlx0Kc9e9gTVVifXVVXwtsjibTnLwNfNomkFfLh/EEVReLepByFgeW0JMSXGc03PccWzV/AJ/ya+UmTh5rdu5rfv/5aHtz3My3teZp93lIPt1BLSGkN31hLSbl+QUDSWMTQEYA10gascc0EVMbOLqvDIA3ZyIYSsG883R9Dl7+J7L3+dT7sitBkNzDNey98+8jeOrjiafz/635lfMp8fvP0DuofSejy0ksw8wi07+nYw26YaY3sRpU4rff5QPKnvNDvxKtGs5aMpOYJ9awHYLObGwx6zi2TlkNa0lszsCifN3X4UmztnGCscC3Pj6zcC8PhHH+e2U2+j09/JVc9dxQ/f/uEwb2MkFEVhb6+fhQUhOfc5jxyB1WTEZTUlPAIYXXgok87QAfQITCMfcvjhD8XobJ/Pfy79KMfM7+K+Tfdxz8Y/YJ8JD3fcz4uPlLCgdAELShawqXsTq9tWc8MxN/AF83QE9yfKR0Fq6ueQmfj6MV/n5T0v8+N3fsyDFz6Y0uWbcW1hPy80v8C5NefiGOonhhGvwcXJDcPDDcU2N4MGEwMDvSQ7r01dPlxm1SW3ZzYE86vcXGO4hC86iwiLl/jXhj/yiTfukPLaiz4Gm/4O8y4gYi2ipcfHOQsTMe+9nr38v/X/j1Omn8Ilsy+RV5747/Dh0/Dcf8pqIrXmfJprGvdXnsl1Ox/kaxvu4BeF0zhrVlqJo6sC9r/PwoUF/G3NXtoGAqze3cOCajdrO1/ndxt+x87+nTQUzmZ6cIguWxHb971JT6CHmJKoMppbPJczZp7BGbPOYGHJwtwzAIqlRzDf1kfbQIBAJEC7r53uoW7KHeVMc02LN5NNyxQaUhTwdnHCMRfzyjlnYLivIV5C6g/72di9kV39u/CGvPjCPrxhL96wF3/Yj0Bw+bzLOXX6qcPWWOgwj1y5BqzvWM83V30TT3CAa/sHeHXge8xfejIGIfd2FqOF2069jSuevYKb37qZu866K/Fcmq7+CLvsHX072NG3g49WqEqm9mLK3EMoCvT6Q1S4bbgtbsIoBEM+Mp22UgxB63tgMPF+ZBblamlkpaOSKmcVr+x5hU/N/1TKfWeXuwhFY4RNBVhyTAL87fu/ZWP3Rn55+i+Z4Z7BDPcMTp9xOr//4Pc8uPVBXmx5ka8f/XUun3d5/P3JRb9fdpM32NUijDRDEFNiGR+nxGmR3lxyCemsE0Z8PkBWI6pe6oHuKoYj1BBow7HnVxVw0rQ5nDTtJHxhHxf+/mEKizpZOt3Ltt5t3L//fgBuOfkWLm24FN5XewMcSXFrZwV4speJFlgK+PaKb/Pt17/NQ9sf4qoFV+Vc28t7XsYX9nFJwyWw5v8YFG6WzSyRk6fSKLS58BqM+DypX+jd3V5OdcgPU7YcgcloYN7MKp7xf4y6mbt4pnQJnyhfKXf0q34GKLDs0+ztGyIcVagvkzHemBLjh2//EIMw8IMTf5A4uRhNcOnd8IfTpDH45APx5yrra+VPXhNfnbaAb636FrecfAsXz7446T0sB28XC9Uu3PUtfWzofpeKWa/yjVW7qS2o5Ren/YJznTUY7j4BTroFln2KaCxKb6CXTn8nazvW8sqeV/jjpj/yh41/oNJRyRkzz2BJ+RIisQiBSIBQNEQgKn/7Ql7aKyvYan6dTt+bvPBgap7FKIwUWSqxz3TxfPt7dIg5lNvLKbYVU2QtoliYKYwMYS6sJKD4WVNUzrr+7ax77jNs6d5CRBvqjvREnWYnTrMTl9lFT6CH61++nqVlS/nqUV/lpGknxd/HYodlRCnqR3c8yk9W/4Tprun8KVZOXXQr/29oBqc6UsNls4tm863l3+Knq3/KQ9sf4sr5V8ob8qwaemT7I1gMFi6xqtsMWxHl6ojLbo80BFrs3xMLYY1FIW2jow2ud1lUQ1C1hP0emFclT3JCCC6bcxl3bbiLPYN7mFUwK7F+1Qv1G5xYsngvb+17iz9t/hOXz72c82rPi1/vsrj4rxX/xccaPsZP1/yUW1ffyhv73uBnp/4sZUBOJvb0ynxHrUV9TrWZbHP3Zn6z/jesaVtDobWQUlspJfYSSmzyRxQNscc3j3DBxzAjRldC6uuWoyhJNOzphmCS2dGuaQwlPhBOs5Nq2wKi/vncespJAPETR4FF/eKkdf8BcjfbtiHn851fez5P7nySu96/iwvqLqDEVpL12Kd2PcV013SOrTyW0OCvaI+6U8pGk3GYHXQZDAT9qV+Spm4fH7EG4sdkY3lNMXet2sUNJ13IHzffxb6Vv2D68dfB4H7o2AINZ7Nrq4xzzlYT1Y/ueJQ17Wv4wYk/oMqZFjutWACn3wiv3CINyqKPyeu7tlNYNpd7zr2HG165ge+9+T18YV9iB+gsZyDioz+8FkvZy9yy/h7M05sQxipuPeFWPlL/EUwGk/Q4IJ50NhqMlDvKKXeUs6hsEdcsuobeQC+vt77Oq3te5cmdT/LQ9oeGvW6jMOIwOai02imLKPQHl/HFE4+lyllFqb2ULn8XLYMtrGraSqdxN2+0Pc8Lex8b/gbWzcLV9H/4d91HTIlhsiosVhSuWXQNy6uWs6BkAQWWAsxpg0TC0TBP7XqKezbew5df+jLLypdx/VHXc0L1CRTZzVlzFuFYmNvX3M5D2x/i5Oknc/vJP6PgN8sIzbuYWKfIuFn41LxP8UbrG/xq7a84ruo4GYoxmsGceyaBP+zn2aZnOa/2PIr8AXm8yTKsqCIxk8BAWciXMDIqWqGEw2CFfevhqE/TtSbIKUke7sfnfJz/+eB/eHTHo/zH8v+IXz9brVIbVBwUZVhrl7+L7735PRqKGvj2im9nfB0NxQ3877n/y9+2/Y1fvPcLrnruKu48405qC2uzvvYW1RBUG/oBaCbKnav+gxdbXqTEVsLVC68mEA3QG+ilZ6iH7b3b6Qn04LF66AVO+ftvWT59Bsd3vMUJfTuYUzRnmOcXjoUJRoKEY2Ei0TDhYD8Ri5VIfxONfTuw2Xyp3fSTzJFpCDq82M1GZhSnuvylLitbk7pbLUYLFmPSLsvfI+PmyUqKrgppzWOxrJIHQghuPO5GLnvqMu5cfyc/POmHGY/b793PmrY1fGXZVzAIA57ednoVd8b8AMgvYcBoQIS8cvKY1UQoEmNvrx/LzED8mGyctaCSO1/ZiTO0AoBndz3Ll5Z9SVZJqJUSTd1ypzy7zEW7r51fr/s1x1cfz2VzLsv8oCd/Qw5P/4caInKUQncjHPNZnGYnd519F/+56j/5yeqfsK13G/6Iny373mFPzQx46z+wlgk8wQrC/R/j0etvpNyVVLKqqY7mEJsrsZVwacOlXNpwaTzcYzPZsBgt2Izyt8mgfuz/72Ps627jfN/H+OLS84Y91sD+D9m6bw/v/Ohc+oJ99AR66A/00xfso3//evrW3E3fwosprFjMsX4fS1/8CfYLHoHKhVnXB2A2mvnE3E9wyexLeGLnE9yz8R6ue/E6jqk4Bp9lET2hSiKxExPrBPoCfXzrtW/xXvt7fG7R5/jGMd/A2PoeBAcYmHYqIL2JdIQQ/PjkH3PZ05dx4+s38teP/FV+pkcYAfnc7ufwhX18ct4n4e17ZEULCVlkbdeq7a7jUtRphsAT8uA0OzH27oKQl3D1MQwGIinyyhWOCs6YeQZP7nySrx39tfh3rshhodRpoSdqY1YgdfhPTInx3Te/iz/s577z7sOWoYM++T349IJPM6d4Dt9a9S0+/Y9P8/PTfs6pM07NePxe1RDEwq38qKyEJ16+DovRwleWfYVrFl2TqIBK44ZH3uKt1tVcfJKf1dse4/VgCzx9GcXWYmwmG8FokEAkQDAaJJo+mn1GBXT+C576FwDmWsHV/3yGs2adxVk1ZzHTPUZdrTw5sgzB7tfBaGFHh2BOpWuYjkfZSMJzflVeItm6uyplQmmoN+fYxPrCeq5ccCV/+fAvfHLeJ1lYOvxk8cyuZ1BQ4mGTqLcLj3Emx0/PXPkjB9iDkwBb2wZZUVvCnl4/MQWMhmD8mGwsnVFIdaGNdxsVllcu59mmZ7lu6XUpu5ddnT7KXBYKHWZ+tOp2IrFIakgonXiI6HR47r/g3FtkIlE9eVuNVn59xq+5+a2beazxMSodlSxxzeDjO3ez5KzbuG9THc9v62dBdUGqEQDZQ1AwI+PQlUzYTLacOz+KZlGy5308wUjckCbTNjBEdZENg8FAqb2UUntSbsg3BP2DsPQrULUE9m8A5VaZJxjBEGiYjWY+Oe+TXNpwKY81PsYDWx5gX3g9lMNJf7uLJWVLWFa+jNlFs/nt+7+ly9/FT0/5aSKstusVEAbay04ENlPszCxQVmYv48cn/ZivvfI1fvv+b/nW8m/lFJ5TFIVHtj/CnOI5LCtfJoel24uARJlkt0d+T+KhoSxTyuLKo6oIW59tFtA/LOxx+bzLeWnPS7zU8hIX1l8Yv352uYuOQRsoqV7vfZvvY3Xban500o/iCeeRWFG1gocueoivv/J1rn/5er5x7Df4t0X/lvJZbve1807HixTOXM8lHWuIuJx8ct4nuW7pdZTZc5cFV7lKGOxZwPePPx/R0Ubbzn/x7kdvZ33nemJKDJvRhtVklb+NVqxGK2ajGfPQAKZXbsV8zDWY6k7jf1btpjfUSjC6k1+t+xW/Wvcr5hXP46yaszhr1lkZPYzxcmQZguf+C4rr2N5xbcZwS4nTysBQmFAklrlDM0kvPI5TfRxvx4jzc7+87Mv8o+kf3LbmNh44/4GUf6aiKDy16ylWVK1ghnsGsZiCLdSHrWjFsK5WDYfZwZBQcDHEa60DrKgtoalL7uBjqKGhDJ3FGkIIzltUxd/W7OFHKy7kJ2t+zObuzSwpXxI/pqnbS32Zizda34gn3UbcnVQukiGiV29NlCom7eLNBjM/O/Vn3HTCTfJE0roO1j0H1jLWTa/i+U0Z+gdgfFPJMlE0C3u4DzsB2gcC8e5QjX39AaZnKR2Nq86qndGUyjLIdPG5fLAYLVw5/0qunH8lP3vhHf533at86kzY2P0B922+j6gSpcJewf3n35/yv6HlbaheRk9UrrHQnr2k9vSZp3PFvCu4f8v9nDbjNFbkMASbuzeztXcrNx1/k/yMDvXFPQK31YTFZKDbl+oRZGpshCTlUbWgokspJJMhOKH6BGa4ZvDIjkdSDUGFk30dZmBAJuiFYEPnBn73/u+4oPYCPtbwsdxvbhrTXNP48wV/5ua3b+aOdXewrXcbJ1afyNqOtazrWBevPjM4bZwtnHw1aGDm8d/L67FLnBZC0RjeYAR3cR3Vnk4+VnMuH5szwhpb14HXBzNWQt153PfC20w3HM1DF59Iq6eVl/e8zCt7XuH3G37P7zf8nlc/+WrqpmQCOKLKR6lYSLTjQ7o8wbi0RDLabidrHXdSiVccVSsnn5rhAksBNxxzA+93vs9zu59LuS3eO6BW4Wxt7aYAH2WV2ZtZnCYnIRRshiCb1fp7TaIhpMiu0vQRgemcu6iSYCSGJXg0VqOVZ5pSewp2dfmoLTfzk9U/oa6wjs8t+tyIrxOAU74hdfHX/UleztBDEPdWNAPq62TpDGk4Tpqd9j7HYjLENN6O4mSSegkylZC2qZPJMuLtlGJz2tqtLlldkjaXYBjhAGx5Mmuz0YyCaiKDy/jS4m/x8EUP8/aVb/PnC/7M45c8nmoEFAU6NkPV0ni5abEjt2Txt5Z/i5numdz81s34La6s5aOP7HgEu8nOR+o/Iq8Y6ot7BEIIypyWuEcw0gD7uPKoagjaozJ0lG4IDMLA5fMuZ13HupSegtnlLjpCVul1h3z4w36++8Z3qXJW8d8n/veYdsYOs4NfnPYLbjjmBp7f/Tw3v30zb+57kwUlC7hxxY04u/6LM2x/4GdeRhWSSXQXJ5eQNo98R79a3queW3zBKE6L3KPPcM/gmkXX8MAFD/DKJ1/h1yt/PeFGAI40Q1C5EONAC06G4mJzyZTlaCoDUpVHNbSxi3k2j1zacCkLSxfy63W/xp/Ulq/1DpxTcw4AO1vknICq6uwiYtpJ3mAMxufjNnXJUE4gpoaGTNlDQyCHahQ7zLy+zcPKmSt5fvfzcQntPl+IXl+IbuNz7PPu46bjbxqW+MyK0QyX/h4MJukVODPnOeQLUY2pr4uTZpfy4LXHp5SrysXsljHoPMMueaEaghmia5gKaSgSo8sbzNpMhrdDfhaSq2RKG6SxysX6P8Pfr8laYFDo0PSG1OlyZgdHVxxNYXpjoKdNnqArF8ePLcqQI0jGbrLzo5N+RKu3ld8aBjN6BAPBAZ7f/Twfqf9IYhMR6E+UnCIbnYYni0XGXgJvyKsagk6wFdKhfuQzjWC8tOFSTAYTf9/x9/h19eVOPKhebXCQX6/7Nfu8+/jJKT8ZsfonF0IIrl1yLU9e8iRPXfoUqz65ijvOuINPzv00HT2lzCp1yfc4j2YyjZTuYq2ENJ/KobjOkPSCfaFIYixrEmX2Ms6uOTvv9YyGI8sQVCwCYK5ojQ/yTmZEmYlMHoF2gsvQXZwJgzDw3eO+S6e/k3s33SsfNrl3QK3y8fXKHZS7JPsHUQv7RESInZ0e/KEIu7t91JU58UVHzhGALCM9e0ElL2/t5Pyaj9AX7OOtfW8BMixksHSyfvAJLq6/mOOqj8vrNcapWgwf+bXsMci1czPbZfeotwshBCc3lA3f6bV9IH9XLxvdGnKhNpXNyOARdAwGUJQsPQQgQ0OuNGNV2jBsQM0wGl+Qv7MYDE2KesSmsg51jkTlorhaaaaqoXRWVK3gU/M+xYORDt6PDhcrfGbXMwSiAT4595OJK5NCQ0DKWFenRcsRZPYI4jkCbwe4KuP3006ayZTYSjin5hye3vl0XCdpdrmLQUU+x9t7X+fh7Q9z9cKrObby2BFfaz7UF9VTX1gf/7zt65d9EnWFRpn3c+cvHFjilOePeHcx5OkRaIZAepe+YGZDMJkcWYZA3U0us+yjqmC4y6+5dhk9gmhE7ozS8wC2QjBaR9VOflTFUXyk/iM8sOUB9nr2pvYOqAQHpSGwFGQXL9NO8kMCrEqQrW0emrq9NJTa8CpRzBjy2sGfv7gKTzCCGJpHia2EZ1TJiZ0dXqxVT2Iz2WWCcSwce43sOh4JZ1luY9r2gZSxLl8wtnVkwlUBJhtzzN20DaQKtO3LMJAmBW/ncC+nbI48eWSYXwxIPZ7dqq5TFoOREJ4byRCocyQqF9LvD1FgM2XNJaXzzWO/yTSjg5vdZgLhxOtWFIVHdjzC0rKlLChV3+fwEEQC8dAQSIFG7TtiNpixGSxZx1UmDEEnuCrp8gQpcpixmjI3Vn5y7ifxhD280CwN5oxiB36Dk0GD4OZNd1NfWM+/H/PvmV/Y7tfh9vpxjY9t6ZHGrN6u9pWMxiPQQkP+kDSc1sL8uov9qkS7Va3ACkZwWXM3nk40R5YhKJzFkLCzwtGeMbZY5kwtjUthSG1TT/cIhBjT7OJvHvNNjAYjv1r7q5TeAY2YaghyhVQ0Q+A3CFwEeGdXN93eEHOLFXmdMXeoQOPkhjKcFiMvbe3h/NrzWbV3FYOhQV7e9zwmZxPfOOaGSYlLppA29nMY7Rtln8J4NYaSEQIKZ1Jv7qV9IPV/rhmG6izyEvLEVpF6XTxhnMUr2P06RIOAyOoRaCWgI3YXd2yRFVT2YvqHwokRonngMDv4YcVpNJtN3L3+zvj1azvWsntgtywZ1Rjql7+TPQK3lR5vKC7FkkuK2hvyqjpDHXFDkEs64djKY6kvrOfv22V4yGgQuAtL+XlJMd3BAX56yk+xGrPcf996eVLd/Xqe78RwtNLRfCaTpZOSIxACSmrz9wjUasRoTCEQjuGw6B7BpKEIQaMyg/mGvRlvL7CbMBlE5iH28YROhmoWZ3neoSGNSmclX1zyRV7e8zKr21ZzyexLUtrWnb49xDDEwxeZ0AyBz2BgujPK0x/IDufZBTF8BgOObF+YNGxmIyvnV/Dih+1cWHcRoViIx3c8znuDD2CO1PDJeZeP6rWNCbW7OCOKIj2C6uxD2cdM0Symiy7aB1M9gv0jyUv4chiCbHmCxn/JxqzaU7Iai8J8pag7tsjqLOQgm9HOtj2hZAGfGPTwwLa/srFrIyA7id0Wd0qHbnwDlGQIStXqmMEh2T2dTYo6HAsTiAbk51TzCLy5dfaFEFw+93I2dm9ka89WuYTSLp52u/hi1SksKluU/UV52uXv5jfzfRuGsafXj9VkoCiift/z0BnScFiMWE2GxPmjuDa/HIEvEXL2hdT3VA8NTR5d3iCbIzOYHtqdsWpDCEGpK0svQdrgiBRcldlPYjm4etHVzHDJZHCK5AJQHmim11wN5uyNMpoh8ArBknITOzrkjqzWFcUnBE5j7qlKyZy/qIpub4igbxq1BbXcsf4OIoqHhebP56XPMm6c5dk9gsH98v2vPmrin7doFhXRjmEewf7+IYod5vhEuBSCgzJc4kwzBEU10sXPdJJXFGh8EepXQuViWV2U4TPoVhVIc4aGIiE5rlM1BAP+0IiJ4mHYCvmP3n7KbcXc/NbNtHnbeGnPS1wy+5LU5ixN4ycpWaydyLUSUpfFrYaGUnMEPvWy22CR3oKrIj6LNxcXz74Yq9HK33f8nd5ALx+YnmVBMMTnC5fkvF9c6qXl7dzH5aClx8+sEgcGr2pURmEIhBApYTOK62T/xEiS2P6eRKI4KA2BniOYRBo7vGxXZmEL92cViit1Zhlin5bQSWEE4blsWI1WfnH6L7jp+JuY4U5UBymKwvRoK33O2pz3T4SGDCwqlaEug4Aqa0jOKx4hUZzMynnlWIwGXtjSwcWzLyamxIj0ncSyiazSyYWrQr7H0cjw27REcdXkeASu6AB+7wChSELAri2H/HTc6Kcni40mWS2SqZegaxsM7IE558hJbWGfrEpJQwgpFdGfa25x9w6IRVI9ghFKR4dhLcStKPxw3mfZNbCLz7/weSKxSGpYCLJ4BKoh8CQZggxziz1hmYx2aidCNVk8koZOobWQ82vP5x9N/+Dmt24mxBC3dvXg7xtBRVTzCLq3j2ljBtIjmFXikEbFaE153flQ7LTI4TQgPwuxMAyOoIybVITiC8r3yqnnCCaP7e0edijqCVerukij1GXJPKXMl1rrm4KzQoaOxjAMY3HZYq6Yf0XKdf3eAHW0MVSYu2MyOTQ0p0gagpklDixhLz6DwDEKQ+C2mTllThkvbGnn8jmX8/H6zzLUdS71ZfkPXx8XznJASRjcZNo3AkJWIU00xYlegk5PonJof/9Q1oE08TCgK0P+prQhcy9Bo5QOkIYgdy6hyDGCFHW8Yki+H/3+UEZ5iZyojX6nOGdxyexLaPW2clzVcdQV1qUeFzcERfGrtKHq2lwCl9mlzi1O8wjUclJ3RL6WgLUMfyial5jaJ+d9En/Ez2utr3F5/bXUhGCgrzv3nQbboET9zrS8NeJzpKPJT88scUij4q7KXe2WgRKnJTU0BCOHh/w98SKUuEeg5wgmj8ZODx029YPSmXk+QKnTksUjUCtBMuUIXJWgxDKfxDR8PYkv1Qj0tu3EKsLESubkPC5hCAS1brmbrStzQtCDT2SfTpaN8xZV0to3xL5eAyeXXA0xa1xsbtKJl+Fm2Mm1fSArciz5G7a8UXsJZorOlBLS/f2ZR1QCCe8vPTQECUOQvilofFGeuAtnyNcCOUtIc04p69gsNa9KG4hEYwwGInmVjqaQNJzmv1b8F8dXH88Xl35x+HGZksVpekMyR2AYliOIK4+G5PvaI4qA/HT2l5Qt4ZiKY1hRtYKvHXstg9gZ8uT4/iiK9LDmXQBmx5gMQa8vhC8UpabUAQOtUDB91I8hzx9JoSHInTCOReV5Ie4RTE1o6IiSmNje7qG8chp4KqEjiyFwWWUdcDr+HjnQw5ThQ6ztDDNVkmg8dKX88l3198y3JzG0TybJzJW5u2hNBhNWgwWfMFBiDlNb6uDomcUQ/ACfIft0smycvaASg9jEv7a0Y1d3JJnmFE8K8aayDEn3to1Qc+LkPG+8lyDRVOYNRhgMREYfGgJpCKJBeSJRvQ0CA7DnHThJLXt0TwOTPWsXcpHDkuKdDKNjC5TPB6OJQfWkM1JX8TDihqCfQmsh9557b+bjhvrkaFBrQkyu2GHBIBLzvWXVEMM8Am06mSsor+9UCoHOvDwCIQR/PPePmAwmDMJAi3AR8uUwBP5eGYYpnAkzj4Pm0RsCTXV0VolDht/mf2TUj1HitCY8goLpsqEyVwnpUD+gxA2BN6gniyedo2cVy47VioXQmT005AtFGQql7eiSEjrD0E4I2SqHAgNSi731vbzmmEbVAe3u6SPH551mBz6DQIS8PP+N0/jamQ0QGJDzipMSfPlQ6rJyXF0Jz29pp6nLS7nbSoFtlCeYsRL3CNLcf18PDLZOTn5AfV7FZGOG6I7PLm5Tewiyy0t0SHmJTJ8HbbefnCfY9aqM6c85V142GHI2n404paxjSzws1JdnV/Ew4jMJRphSFuiXRiMpRGI0CEqclhSPwCcgGkwtH40PpQl4QBhpC8sGyJGSxRoWoyVeqBA2u1GGcsxP0BLFBdVQc4r8fmfr58iCVjpaZx+S3/cxyJmUuiz4Q1EC4ajMGRXNyh0aSpOX8KvnHYeeI5g8/vuihVx7ar1MsnVtzxjT15pChoWHNOXRTGghgmy9BHvXyNDRUJ+sgBkBc18j3UoBZZUjVyw4zW4p+BX0YjMbMRoEscAAfoMBp7VgxPunc96iKnZ0eHm9sSuuB39AcCZ5Vcm0T0JHcTJCQNEsaoxdcUOgNZNlF5xTm8kyTZuLx/+TdvuNL8qT6YykzuzS2VkF6opyDafxdYO3PZ4o1gzGqJPFJpsML400tzitq1ij1GmlRzMEqsyEP5LmEWiGYKgfnOV0++Rud0wDV2wFGMODKWNkU/AkVfnUniz/3vPOqJ5iT480BNOjanl5+fxRLzOllwBkeChXaChNXkL3CA4kFQtl+V9v07CbtIqIYSWkmeQlNFxZTmIayfFKrSM0B67BJpqZnldTidPixGc0QSghFzCkzmgdSWcoE+ctkp2UHYNB6ssPUH4A1A5ty3CvKl4xNELp4DgQRbOoNfbEB8JoIaLqXKGhTPkBkAbCWpiI/8disPNFmH2W3CFqlDZAX4ssBU2jyGHGE4wQjsaG3ZYsLQHkrTM0DCFySlHHyWIIytyJoor4TIJw5mSxy9cbLx01iMTJcjQYHUU4Y774YPdhaBssdxVMP1YaulGGh1p6/VQWWLH2qf+7MSjdDjcEtblDQxnkJUAvHz0waCWRGSqHNA2UYU1l/t7MpaOg5g5s2UtIW95OSCO0bxxxeSVDzbSZszeSJeMwOaQhSHLLveqXO9d0smxMK7LHFUDTB9ZPKkKovQRpoaG2D6R7nS0sNxEUzaKahEewv38Ig4DKbDtXb0fmiiGQr6N0diLs0/6BPF4LC2mUzZGKmhl2i9ruPmPCeJghyE95NCO2wpEH2CfNIkim1JmkN6T1s2RIFpsNZqyqLlOXJ0ipy5q3FEbKUl3FuIWfpq7hekbyyVSPwFUl83gzVkDL6BrL4qWjXTtkwrkgu+BjNhIRBfX8UVInjW22QpG0/iTNEDjMemho8imfL2O8GSqH0isi4vi6s5+MhJA7xEwVLyG/bH2fe56sUGkfwSPw9eCKDdJrr8nnleA0O/EZUkv3fCFpCHJNJ8uF5hUcsESxhrN8uFfVtnHy8gMaRTUUxAYZGJBf1v39ASoLbJiMWb4emQTnkimbkzAEjS8CAhrSVCNzlJAW5hKe69gi3yc1uT7mHAHIDcxIHoGvG+zDP/fFSSWubrPqEURTu7MT8hKd4B5ZXiIXrqJSCvCzq2u4jAUgK4YcZQkJkpqToH3TyK8vib29fmaVOGUfQtmcrBMHc5HwCNTzx0glpMOUR6M4LMZhQ7Mmm3EZAiFEiRDiRSFEo/o7Y/eFEOIa9ZhGIcQ16nUOIcQ/hBDbhBBbhBC3jWcto8Jsh5L6nB5BT7JHEPJDZCh7aAjUprIMoaF9a2U1Q83JMrzRvin32rplothfkN/UJZfZhd+QqvPiD6qNPKPoI0jmihUzufrEmszDYSaTdKmOwCD07pqcjuJk1Mohq6eVWExRS0ezhIUURR1ClENWu7QBBvZKwbbGf8H0Y4Z7EKXq/zeDIdBO6gOZmso6Nse9AXlMGIOQHcmjZqTQUCQkE/XaySxtjYOBMNGYklAgjaau1xv2ys+gLyEvUTbGgezOglLsIsTu9v7MB3jaUnWBak6Webk97+b1+IFwlPbBQMIjGOPci2GhZa2ENEMYGpDFEGanPCcxNcqjMH6P4DvAy4qizAFeVi+nIIQoAX4AHA8cB/wgyWD8UlGU+cDRwMlCiAvGuZ78qViY0SNwWEzYzIZ4IgzIPLQ+HVdlZkPQ8jYgYNbx0hD0NmWU69VQuuRc3khJQz6vAofZIUv3gokcgZakG0toCKRX9ONLFh9w4av4/GcNLZ8yGRpDyai9BJVKB73+kBxRma1iKDAA0VBuj0Db7e9dA61rh4eFQMbdHWUZE8bF2RRIY1HZoVyZaKzrU+UlxrSDHMkQ9O+RJ9OS+oxrVBRpiDSPwBcLplTFecNeXEa7rJhyVdI9Do/AoJa7tndmCb962lLlIGaskHIfefYTtPZJ+en6gpg0fhkGKeWDW1WBjYeWy+bI93nHC5nvkJZ7zDQy9UAwXkNwCfCA+vcDwKUZjjkPeFFRlF5FUfqAF4HzFUXxK4ryKoCiKCFgPTD6oNxYqVwk3bUMJ2UpM5G0u8mlM6SRTXiu5S1pAGyF6hdYydrDABDq2EZAMWMpHUVoSCgpHoFP1XIfa2hoytD0hrSTyWTMIMhE0lyC9oEA+wfyGFGZrV8EEoZg9R8AJbMh0I7L0EtQZNcm5aUZgt4mWeSQ5BGMRXAujq0gd/motovNYAiKkgboaANsPAaD9IJUvCEvboNcm+KsGFFwLvdapSHo6clSkDGYNkTG4pBJ4zwTxlrpaINBTTqP0RAYDIJiR1J3sckKiz8BW5/JbHTTytL9amjoQDNeQ1CpKIommNIOZNomTQeS5T5b1eviCCGKgIuRXkVGhBDXCSHWCiHWdnWNTUckhYqFgCJ3WGmUpQvPpdX6ZsRVKf+pySWpkRDsfU+qTUKi8iVHwjjSsZ0mZRqVhfnt5p1mJ0MoxJI8Ar9qCMYaGpoynOVyt619Ydo2yvd1FJrwY3veMmJGGzNEF1v2S82hnD0E2lqzoYV9tj8nj8sW2irLPNEsqwJpfAZBUmhoLDpDGiN5BDkNgXzOPn84vuHwpg2w94a9ONVTjNdcQjiqjNsQ+Ad7ZY1+MtGwNNDpQ2RqT4b976cUUmRjj2oIpkfUU9U4RqKmdBcDHHWVDC1veXL4wUnyEiA9goMyNCSEeEkIsTnDzyXJxymywHfkbqnhj28C/gbcqShKlkAaKIpyj6IoyxVFWV5enuNLmC/alynD7rzUlSY8F5eXyGUIKqQbnRzaaNsgPwA1J8nLRbNkaWGOElJj7052KdVU5PmFiQvPaRUbsVh8OtlYQ0NTRtLISkB6BJOdKAYQgljhLGaILta39AO5Skc1naEcoSGLU5UnUKDhnOxJx9IG6UWmnYzdVhMGkaFqqGOL7PJNOkn1jUV5VMNWKD+fGUpYAWkILO6MIdHiJI/AbrJjQKgyE0mGIOTFrZ4ReigCEuNgR79W2RPjwh+fy514ok5AGb5hqDlJVmbtXT3iw7f0+LGbjbg9TbIbWBs1OQZKnBb6kg3B9GPk/2zDX4cfnBYa8h2soSFFUc5WFGVxhp+ngA4hRDWA+juT37YPSJ4APUO9TuMeoFFRlN+M+VWMheJa2eafIU9Q4kz3CPIMDUFqeEjTRZ+lyiMIIQ1Qtsqh8BBW7152xqZTUTA6QxCv4Q555c6MQ9QjAPnFDgektzbZYSEVY0kNM0U36/fIyqGsoaG4IcgRGoJEeGhulrBQ8jFp4SGDQVDksAzPEXRskTHnJGny/nF5BEXyd7YS0r7d8oSYQXhNMwR9/jBCCJxGqzqcJnGS9oQ9ceXRDkU+11hzBJpHkLFySFNxLUjzCGYeLw1nHnkCrXRUdO+QwnX5zubOQInLklp+LgQc9WnY++7wUKC/d5ghOCg9ghF4GrhG/fsa4KkMx7wAnCuEKFaTxOeq1yGEuBUoBL4xznWMHoMRKuZnrRxKnsCEr1uWm+aSbIgPsU9KZrW8LUtVk3dUVUvkc8YyNAv17EKgsEuZRrk7+xyCZOIeQXRIxtaDHvwGgRGBzZjfYxw0JAvPdW6Ru7nJThSriKJZzDR0sVM9yWQNDfk65cklQ0llCmVz5c6y/ozsx5RqchQZKofs5uFTytIqhkDuyLWcwqjROs+zhYd6m7LujNPDV26TXQ6wV8dVKoqCL+zDFQmD2UFPSB5fMlaPQF1rgfCzqzPNI9AMQbpHYHXDtKPymk+wt9fPrFKHLB0dQyNZMsNCQwBLr5DnkGSvIBKUjaBJOQJfKIrzEMwR3AacI4RoBM5WLyOEWC6EuBdAUZRe4BbgPfXnx4qi9AohZgDfBxYC64UQG4QQ145zPaOjYlHmXgKnlVA0Fm/3xt8jv/i56oq1HaImSBaLytI1LSykUbVYus+Zug3V0tE9hhkU2PLbFcQVSEGqPwYH8Qk5nSzTOM6DmuTQUJuaRzlAHgHFNRTixaXICVVZu1+1WcUj1Zif+h/wmcczNmPFKamTJ4dMvQQOc2qOIDAgq3iSDEEoEsMXio6tmQxShOeGEY3IzucM+QEgPiNZ81pcJoc6wF4a0qHIEDElhjscAFcFPepxJeMJYwEz7GGautM8gkHNEGSQZKk5GfatS0lip6MoCnt6/dQVmaTxG4O0RDIlTgsDQ+HUzvCCatld/sHfEnnEDJGGQ9IjUBSlR1GUsxRFmaOGkHrV69cqinJt0nH3KYrSoP78Sb2uVVEUoSjKAkVRjlJ/skggThKVC+VJJ22IRbyXQAsPpSV0MpIeGmrfJK19zcmpx8UTxhn6Cbp3EEPgc9fkfRJPlqIm6IXAIF6DwHmoeQOg7rKFagg+kF/+ovyqp8aNWjk0XXQzrcie/f33dmbvKk6mYBrUn577GJNVPm82jyA5NNQpFWmTS0e14TXjShZDZo9gsFX2v2QxBEIIiuzmeEOby+xMkaLWJKidIT+4KuMx8zHnMywuEAZmOcKZQ0MGU+bO/9pTZAFC63tZH7rLG2QoHGWRtUvm+caRKIZEd3FferL/qE/LITXaTOU0eYlYTMEfih56huCQp0KVmkhTIi1V45jxhHEunSENq1vmHLQYsuaOavkBjfIFMrSQxRB0GSspKshfPjp5OA0hLwQH8RsMuA61RDFILR5HqXwPtUTxgfJqkuSos84hgHhz1IRRmrlyqNhhSZ1SlqViCMZxcs2lQJqjYkgjeYBO+rjK+FCagAdcFfT6QritJiymMZ5yDAawuqmyBtnV6SMWS6pL8bRLaYlMXtrM4wGRs4xUKx2dHS8dHV9oqERtKhsmUzPvQml8tfBQmkfgV6uhXAdYeRSOdEOQpXJIs+jdyR7BSFo3QqR2F7e8JRPShWnDLcw2mfDLVDnUvYNmpuddMQQJYblkQzDa6WQHFa4KucPr2HLgwkIQ9zxmiK7sk8lADQ2NkCgeDVovQZqqZmH6lLKOLfIkkjQspW+syqMauTyCvAxBwli5LAUp5aPamEqXvx9cVfT6QmPPDyStt9wcYEjtAo7j2Z/aVZyMvUh64TkSxlrp6LTwHkAkcjdjpNgp/x/D5pqYbak9Bdl0hg50IydHuiFwVUi3bJhHkCY8l49HAGp3cYf8Ure8PTwspJFJaiIWg+6dbI9UUVmQf1hHa+/3iURoaCzTyQ4anGVSPjgaPLCGwFFKxGhXPYIc8hK+rpErhkZDaUPG+cVFdgueQISIFmfWZhAkeUhaDmHUYyo1chqC3dLDdWXv4Sh2mOnzqR6BtUDdjEhDEB9KExiUoaGxjNPMsN4ig4z1p4SHtLGS2ag9RYaGIpmVS1t6/AgBhd4mKJopm9HGQVxmIt0jgNSeAl+qIZgqCWo40g0ByDxBmkegJQp7vEF5gs6lPJqMJjzXtR2GerMbgsrFMlaYPDhjYC9EhvgwUj2qpptMoSGfQeC0jH4WwUGBsyJxYjoQPQQaQhByzWCG6GZaVnmJflVeYoINAQzLE6QokCpqN/qwiiF5Eh71mEoNNe6e1SMoqcuZFJclrqpHYCtWO4vl7jo+iyCmxENDY5GfTsFaiEuRj5+iQpouL5FOzcmyI3vfuow37+n1U11gw9jTOO5EMSTOH8NyBJDaU6B5BKrM91RJUINuCGTlUNe2lHJOq8mI22aSoaFAvyxjzMsjUENDmhuaXjGkoQ1hTw4PqXHiXbFpowoNWQwWTMKoJos90iMwGEY9pvKgQTvJmuyJaV8HCEtZLQvtfZwyJ4vR18J+Exkaik80y2wI+ofCsloo5BluCNSwTPFYT7BCyLLMTH0EvbsTgmlZSC5xdVkKCAtBMCiNiuYRuGMxcFVOjCGwFWKODOK2mhIeQcgvDVkuQ1B7skwm73g+4817e/3MKrbKqr2y8eUHIKEVNWymCaT2FLSukSXp6pwKX1DmCJx6jmAKqFwodzFp5ZzxWuB8uoo1NJmJ3a/JdvcMqo1AYqebHB5SS0d3KdOoGEVoSAiBw2THpzXzBAfxGYyHXjOZhladVbU48wSwScRUUsssQxczirOEBvJtJhsN2vzi7nRDoHXuhhO9LhWphqDPH8ZkEOOrO88kMxGLJZrJclDslGMZg5FoXG/IqxoVzSNwxmIoE+UR2AoQAQ/1Fa6EIfDkKB3VsBdD3enw4VMZR8Xu6fVzlHtQhiPHqDGUjMlooMhhHp4s1tB6Cna9klKNGPcI9BzBFKB9udL6CUpdVqkpHlcezcMQOMsBRWrQ15yUveLFVSF3lckdxt3bCVkK6cU9Ko8ApLicbObxogQG8QlxCBsC9SR7IPMDGkWz1CEi/Zlv902CITAYUgfZaEuxa6GhUJIhWJByjOwqtoyvXySTIfC0yVBKjkQxJHktyXpDqicQNwSKQsBaSjASm5AcAYEBZpc72dHhlQ2f2ZrJ0ll4iRwClJabC4SjdAwGWWxRB9uMs3RUo8RpyW4ItJ4CSO0hCOmhoamjYj4gMlYO9XhD+QnOaWgniLA/e1hIo2oJdCR7BI30OeoAMWpD4LCoMwmCXgKBAWLiENQZ0tD6MQ5kfkBDLSFlYG/m2/PRGRoLpQ3D5Kjjom6+sAwhFteBNVVNtt8fGnvFkIatcHj5aB4VQ5AsMxFKGAK1Wsgb8uIUZgxADzJMWToBoSGCgyyfVUSXJ8jOTm9iMlm6vEQ68z8id+EfpoofxAfW0yqvGGfpqIaMKGQZqwkyPATDJKhBTxZPDRanDOFk6CXo9oby0xnSSD5BZEsUa1Qths5tCcGv7h20m2dhUmVsR4PT7MSrzi0e73SyKWfGcrlbyibdPJkUq81rncMVaQFpCAym3FIjYyHD/GJNNmLAH5AKmmn5AYCdnV5qSsZp8DN5BHkagqKkSWrx0JDaP+AJeXAJA9hL6FUrPcecy9CwFgAKK+tkVdeq7V35ewTOMlk9lBYe0kpHq0J75Pc3w3zmsZDTIwDZU2AvSSkH9qs5AoeeI5giKhfBvvdTJKRLnRZ6fUFivlEYAm036ygdOdZYuUR2bnbvkHkIXxctYjrlbuuoh4w4zU786txiv5qsO3RDQ2Xw2cez14VPJlVL5Rfzg79lvj1feYnRos0v7m+JX+W2SQXSuY33yusXfSzlLr5ghJ1dXpbMGGdRQDZDYDBDYe7xICkzCeIegSzv9IV9UnlUTRQDlDgnwHsBplmDzKlwsWpHp5SXMDsTukm5WPBR6XklSc+39EhDUOBtmpBEsUaJ05rbEJhtcN2rcOZN8au8eo5gill8GQzsgU2Pxq8qdVmIKRAc7JLJPEseJ1bNI5h14sgdsclSE2rF0I5RVgxpyLnFMlms7cgO2dDQVGIwwtGflUm8vpbht/s6JzY/oKGVkCZ1GBsMghNtezhxzz2w6OPyM5rEh22DKAosnSxDUFw7YrJea5zqS/YI1LnFnrAHZzQGrop4GeWE5AgAAgOsnFfOe7v7iAzsl95APnmSBRcDAj58On7Vnl4/LqsRU2/jhCSKNUqcZvr84dQO6HSKa1O0qHzBCHazEeMBnlcMcOBNz8HIwkuh+jfw6q2w6FIwWeMyE+HBTuxp3oA3GOGOF3fQ5QkSjSlEYjH1t8J1jjMpnX4ZI36kShvAaJXx35jcCWwOVFJRMXqNIDmlTCaLZWu//dANDU01R38GXvs5vP9/Kbs1YOK7ijUyzS8O+biNOxkwlVJy0a+Hneg2tsqT9+Lp4zQE1gJZmhqLJk78vSNXDEHyJLUQbrOsfvHGZFzcF/JRGA1DQWW8jFJrtBozSZIYK+fN449v7MbTtYfiXBVDybirYNYJMjy08kZA5giWFQUQA4MTligG6RFEYwoDQ+G8Q2K+KdIZAt0jkBgMcNYPZL32uvsBKFP/eVFv9zB5iV//awf3vbWbTfsG2N7hoaXHT9tAgD5fiC94vsRv99aO/JxGkyxdbd8kw0NGC5t8BWP3CNS5xYfsdLKDhaKZ0HA2vP8XqcCZjHeCdYY04vOLkwzB899lutLG74u/nTFuvam1n6oCGxV5ypVnRdtla70EiqKWjubODwDYLUasJgMD/vCwAfbesEcqj7plV7HRIHDnqag74loDAyyvLcZhMaIMto0YRozGFB54u5m7V+3kHesp0LmFf7zyOo+ta2Vbu4flDrUIYIISxZBIjPdmairLglQePfD5AdA9ggSzz4TaU+G12+GoT8c9Avy9UJKo9d3aNsgD7zRz5XGz+OnHlgx7mG889D5v7uxBUZSRy/oqF8O2f4DZTqxkNj17Y2P6YmtzixVfN15kY5weGhoHx34OHr4KGv8F8y+U18ViqrzEBEzHy0RpQ8IQbH0W1j/APwuuYLWyIOPhm/YNjD8/AKkyE/Zi+RpD3rwMAchwT58/hNlgxoYBnyI9Ak/IgzMih9b3tocpdphHnfsaRtL8BKvJyEn1JTibu1FcVeR65Bc/bOcHT8tikGpm8Y4Ntrz8F+6OyiGLy6ZphmD8XcUaWs9Ery/E7Dw/Mr5gZEryA6B7BAmEgLN/KMtF3/19/B9pCiR0hhRF4eanNlNgM/Ff52Z2I0+oL6XbG2RXcgt8NqqWSCmKPe8QKJJx4nwnkyXjNDuJAYHB/TJXQEKMTmcMzD1P7vzXP5C4LtAvk/uTERoCOb+4Z6csh3z636F6Ga9O++LwKWWAJxCmqdvHkvGGhSDJEKgeQZ4VQxpFDnNc/M5lsOBRIqAo+ELepK7i4PibyWDYRLVz6m1YCdFjzF3I8eDqPVQX2tjyo/N44YdXEq4+lm9M38pr/7WSl/7jdFaW9MoRshPo7SVkavL3CLxTNKYSdEOQyozlMP8ieOtOihlECLCE+uKG4PH1+3ivuY/vXDA/a9zvhHp57LtNPSM/n5YwDgww4JQx2TGFhjQF0rAHv+qFaMk7nTFgNEtxsMZ/wYA6VXUyuoqTKW2QgoWPfl4OUfn4vbgcjox6NVv2y0TxxHgEaVPKxmAINDlsl9GCT0A47CcQC+FS1GSxLzz+RHGGtZ5eLUN3Gweyq8W29Ph4o7GbT62YhdNqosBmxrz4EiydG6kxdNFQ4cLQvUOGhSZQ8nyYcGUeyFkEUxMa0g1BOmf+N4R9mN7+DeV2gTXqA0cpA0NhfvbPrRw1s4jLj52Z9e41pQ6qCmz5GYKk2vAuq2xmGktoSAsD+YQh7hHYTTmklHVG5pjPyiElGx6UlyejqzgZrXKo5S0471Yon0uRw5yqQKqyeZ88EU6sR5BkCIQRCrN/xpPRQkMALqMNj8GA1yfHtbo0j8A/AfISIA202RFfa5WQ8i9vdWQvS/3rmj0YDYIrViS9ngUflb+16qGu7ROaKIbk0FCOprI0fMEIDt0jOEiomA/LPg1r/sjxNrXb0FHKHS/uoMcX4tZLF+eMdQohOKG+hHebehMzj7NhK4x3s+4xyJrtsYSGtAohn0HgNQgcRisGof9rx0VJPdSvhPV/lhU1k9VVrKHVsM89H5Z/AUg0bA0GUpPWG1sHmFZoo2ysg+CTyWQIimaCKb8Td5HDEg8NOY02vAYDXrUbXyqPyulkE2IItPVqa1W7il/db2IoFB12aDAS5dG1rZy9oIKqZEXZkjopYbL1aRjqk0Z+AhPFIIUrXVZTZinqLHiDEVx6juAgYuV3AIWvh+XkzL1BO39+p5nPHF+TV7ne6PIEUkphV6waIcbWhp8sRe03GHCZ9ETxhHDMNVJuYterScqjk5QsLpsLl/0vfOx/4iEKLfzYnxYe2jxRiWJIJGCDSTmCEVRHkylSZysrioLb7JSbkSHpDbsUQcxaSN9EeQTaeuOGQHYV74sUZPTAX9jSQY8vxKePzzDudMFH5YyCXa/KyxPsEUAe3cVpTNWYStANQWaKZsKKLzInvB2AP64bpMhh4T+zJIjTGVWe4KirYMUX2e8zUOayYjKO/l8SNwTCgE8cwtPJDjbmf0Tmh9bfL+P3BvOESRAMQwhY8omUx9fmDGhSzwCDE5kohpRKHECdQ5BffgCk5HIkpuALRXGZnTI0pBkCayEDgSgxZQKayTRUvSEABttQ7CUIs41V2zuHHfrX1S3MLLFzakMGWfGFl8rfb/xa/p7AZjKN4lEYAkVR8IWmrnxUNwTZOPU/CBjkzvqddsF3LphPYZ4CX6PKE8y/ED7ySzo9gTEliiEpR2AQeA0GnHoz2cRgskpxsO3/lI1/rooDN0OZVAkHjXh+YEbRxDyJ0QQWtzo6sVf+HoUh0NbY5wvF5xZ7VZkTl70kXkc/WaEh4a7mxPpSVu3oSjlsZ6eXd5t6+fRxNZlDuWUNcmZ5xyYw2RKCgxNI6SgMgT8URVGmRnkUdEOQHWcZ62deQ1AxUTW9hk8ck1t3JZlR5QlUOj3BMRuCRI7AgN8gDt0xlQcjx1wjO793vjR5YaEsJIu6aUxoolhDUyDtVWdyjMojSMxNcFnkuMrBQB8ALkdZks7QRBmC5NCQlJdYOa+Clh4/zd2JUOxfV+/BbBRcvjzH93ah7COgdM6kzL4YTWhoKqeTgW4IctJ11Nc4O3In3/n4iaNuhhlVngDNEIytS1QLDfmFwGcw4dRLRyeOsjkJJdnJShRnIfkkq7GxdYDpRfaJO7GCenLtH3XpKCTJZftDuNSpeJ1DcnfuclZOgiFIks32tENBNSvnSQOthYcC4SiPrW/lvEVVuRPqWvXQBCeKNbThVvlsBn1qstulh4YOPi5eNp2nv3c5i6aNfvc1mjxBNKbQ4w2OqWIIZKmoQIaFfMZDeDrZwcox18jfk9VVnAW3zYQQqTmCzfsGxi80l44WbultAkT2yXoZKE4aqelSG77ah2TVkNs9nT7fOMdpZltrLCrzNu5qakqd1JY64uGhf2xsY2AozFWZksTJVCyAY/9NTgybBEqcFkKRWPwknwvNI3DoVUMHHwaDGPMHuKbUQXWhjXfyMAQ93iAxZWzNZKCNq7ThM4hDezrZwcrCj8q6+srhkiKTicEgKLSb4zmCAX+Y5h7/+IXm0kk2BAXTpURynqRIUdukJlf7UBdmRcHiro6XT5ZMVLLYWiA7vPtbZJ+HOodg5bwK3m3qIRCO8uDqFurLnZxQX5L7sYSAi38jO8kngXgvQR7dxVM5lAZ0QzBpyDxBKaubekZ0DTs9sumkfBwCYk6TE7/BgO9Qnk52sGK2ww0fwAlfPuBPXWQ3x0NDm/fL2PikeATBQbViKP/SUUhUNvX5wnFD0Bbsj8tL9PlC2M1G7OOZq5y+VpBNYCBnPgOnzysnEI7x53eaWb+nn08fN2t8IzwnAK27OOekMhU9R3AYc0J9Cd3eUGLQdhY6BuUIp7GGhkDmCfoMBsLiEJ5OdjAzCcnEfCh0WOKhoU2TkSiGRG3+GAyB2WjAbTXRPxTCpZa+tscCE99VrBE3BOpwGdUjOLG+FKvJwC9f2IHFZOATx+Zf3DFZlKiy2/kkjPUcwWGMlid4p6k353GaR1BZMA6PwOKiyyQ/RHpo6PCh2GFmQA0NbWodYGaJPR6OmTBshTDULwUXR5Eo1ihySq/FbZE9CR6h4Iwpqs7QZBkC1SNQZxXbzEaOry8lFI1x0ZLqiX+PxoDWHJpPd7GeIziMmVUi8wQjJYw7B9XQ0DgkA5wWF51G+SFy6J3Fhw1FdnPcI9i4r5+l04sm/klshYAavhyDIdD0hpI3IDI0VEGvLzRxiWJI9QiEIaWk96z5UgfqqhMmvidgLCRLUY/EVIeG9HkEk4iWJ3ijsSvnfIJOT4BihxmLaex22Wly0mMyAYquPHoYUeSw0OcL0e8Psbd3iE8fN0IlzFjQVD1hTIag0C6lqN1J/SsuDGBx0usPUVc2gR6q1gndtUOW8yaF7K48bhaLphVwbM0ISeIDhMNixGIyxCuncuFTB9c7JyqXMkp0j2CSySdPMJ4eAg2n2UlU3dXpswgOHwrtZgYDETbs7QcmIVEMiV02jEpnKH4Xh4UBfwi7yY5BdSxcRund9vnyH9WYF9pawz5IG1FpMRlYXntwGAGQG0Gtl2AkfKEIVpNhTBIzE4FuCCaZfPIEnZ6x9xBoJFcK6VVDhw9anf7bu2R4cfEYelpGRDu5uirBOnpvslgdTiOEwKnOCnOZnAQjUbzByJiEFEdcKwwzBAcj+XYXT+VQGtANwaSTT56gazBA+Rh7CDSSK4X0ZPHhg5b0fH1HFzWljrz1rkaFdnIdgzcAco2DgTDRmIJbqAULFhd9PpnbmFCPwGwHg3rCHGFW8cFASb4eQTAyZfkB0A3BpDNSP0EsptDlnZjQkIZePnr4oJ34t7V7Jr5sVEOVhhhLfgCkzISiwMBQGKeQJzO3tTAhLzGRFTxCJAyXWjp6MCOF5/LpI5g6CWrQDcEBIVeeoM8fIhxVxtxVrKGHhg5PNOE5mIT+AQ17kfw9RkNQnNRd7DbIv132konXGdLQEsaHRGjImldnsRxcPzWJYtANwQHhxHqph54pTzARPQSQ6hHooaHDh2Qd/wkbRpOOswwuuRuO/dyY7p4Qngvj1AyBo2ziJag14h7BwW8ISl0WfKEogXBuvSE5i0D3CA5rZpbYmVZo409v7eb5zW0pM2g1QzDeZLEWDrIZbZgMelXw4UJRUk5gwjWGkjn6qjGL6qXoDZnkhsblrJp4wTmNQ8gQ5NtL4DuUk8VCiBIhxItCiEb1d8bxTUKIa9RjGoUQ12S4/WkhxObxrOVgRgjBzRcvIhiO8eW/rOf0X6zif17bRb8/RKcmLzFBoSE9LHR44baZEQLqypwU2CYhUTwBxBVI/WHcRjsAblVwTojU8NaEoPU9HCLJYsjHEESnbDoZjL+h7DvAy4qi3CaE+I56+cbkA4QQJcAPgOXI9sV1QoinFUXpU2//OJBbjOcw4PzFVZy9oIKXtnbywNvN3PbPbfzmpR3MLJYn7olKFuthocMLo0FQ7LBMXn5gAohPKUvqLna6Z9LnC1FoN098bbytUE4VU2WvD2ZG4xFMlbwEjN8QXAKsVP9+AFhFmiEAzgNeVBSlF0AI8SJwPvA3IYQL+A/gOuCRca7loMdkNHD+4irOX1zFtvZBHni7mSfe30dlgXXc6oxaE5luCA4/fnfl0cwsOXg9vQKbCaNB0O8PUzjtWOjbgLu4ll7//omtGNI4+mqoWnpAx4aOlXwMgTaveCpDQ+N95kpFUdrUv9uBTCOcpgN7ky63qtcB3AL8CvCP9ERCiOuQBoNZsw4OLZHxML+qgJ99fCk3nj+foRESSfmgyUrohuDw46RMw9cPIoQQqsxEiJOXHM/8trcosZXQ52ue+EQxwKzj5c8hgNYf1OXJXkIaCMeITeG8YsjDEAghXgIyFex+P/mCoiiKECK/Ab3ycY8CZiuK8k0hRO1IxyuKcg9wD8Dy5cvzfp6DnSKHhaIJeBwtN6AbAp2poMghxfFOnHYif5/2d0Dugg9mT+ZA4LaacFiMtA0Esh6TGEpzEOcIFEU5O9ttQogOIUS1oihtQohqoDPDYftIhI8AZiBDSCcCy4UQzeo6KoQQqxRFWYnOqNEUR3WdIZ2poNhhiU9S0+j1hVg2o2hqFnSQIISgqtAWnzmSiamWoIbxl48+DWhVQNcAT2U45gXgXCFEsVpVdC7wgqIov1cUZZqiKLXAKcAO3QiMHZPBhN1kx2nRDYHOgafIbo5LSoCMe/f5J1iC+hClqsBG28BQ1tu9UyxBDeM3BLcB5wghGoGz1csIIZYLIe4FUJPEtwDvqT8/1hLHOhPLVQuu4uxZWR04HZ1Jo8hhYWAoYQg8wQjhqDKxgnOHKNIjyJ4j8Menkx3EOYJcKIrSA5yV4fq1wLVJl+8D7svxOM3A4vGsRQduOOaGqV6CzhGKVCBNhIYmrZnsEKSqQIaGYjEFg2F4pVM8NDSFOQK9s1hHR2fcFDst+ENRghG5u03oDB2cTXAHkupCG5GYQncW8blEsvjQDQ3p6OjoUGhPdBdDsiEYX8f84YCmI9aepXLIHzr0cwQ6Ojo6SQqkaYbgIBgiP9VUF0rZjWyGwKuOqXQdwlVDOjo6OnG9IS1PoP0u1kNDVBZKr6g9SwmpniPQ0dE5LCiMC89JA9DrC2MxGqY07n2wUOa0YjKIrB6BLxjBYjJgnqJ5xaAbAh0dnQmgOC48p4WGghQ7zYhDQA9osjEYBJUFtuyGYIp1hkA3BDo6OhPA8BxBOGWozpFOVaEtR2hoaiWoQTcEOjo6E4DdYsRqMsRDQ33+0OQIzh2iVOXwCLzBCM4pTBSDbgh0dHQmiKKkprJen24IktE8AkUZrpfpn+IxlTB+GeqDhnA4TGtrK4FAdnEnnSMPm83GjBkzMJv16pXJRgrPJcpHdUOQoKrAhj8UZTAQifdcaHiD0WHXHWgOG0PQ2tqK2+2mtrZWT1DpAFL4rKenh9bWVurq6qZ6OYc9RQ4z/f4wkWiMgSE9R5BMZaFsKusYDAw76fuCEaYVjm9C4Xg5bEJDgUCA0tJS3QjoxBFCUFpaqnuJB4giu4U+fyheOVTq0g2BRrV6os80l8AXnPrQ0GFjCADdCOgMQ/9MHDiKnWb6/OFEM5nuEcSpUmUmOrIYAr18VEdH57BASlGH6PFqOkO6IdCoKJDdxekegZxXrJePHlYYjUaOOuqo+E9zczOrVq2isLAwft3ZZ5/N9ddfz1FHHcXChQux2+3x2x599NGUx3vyySf58MMPJ2x9v/vd72hoaEAIQXd3d/x6RVH4+te/TkNDA0uXLmX9+vUAbN++nWOPPZalS5fyzjvvABCJRDj77LPx+0ccM61zhFHsMBOOKrT2yc+GbggSWE1GSp2WYb0EwUiMaEyZ0ulkcBgliw8G7HY7GzZsSLmuubmZU089lWeffXbY8c3NzVx00UXD7qPx5JNPctFFF7Fw4cIJWd/JJ5/MRRddxMqVK1Ou/+c//0ljYyONjY2sXr2ar3zlK6xevZo//OEP/L//9/+ora3lhhtu4LHHHuP3v/89n/nMZ3A4juxZtDrDKbLLE39Ttw/QDUE6mUZWHgwS1HCYGoIfPbOFD/cPTuhjLpxWwA8uXjShj5mLt99+m6effprXXnuNW2+9lcceewyPx8OXv/xl/H4/s2fP5r777qO4uJiVK1eybNkyXnvtNSKRCPfddx/HHXfcsMc8+uijMz7XU089xdVXX40QghNOOIH+/n7a2towm834/X78fj9ms5n+/n6eeeYZnn/++cl++TqHIEWq3tCuTm/KZR1JVYGN/WmhIb+qPKoniw8jhoaG4mGej33sY/Hr33jjjfj1P/nJT/J6rJNOOomPfvSj/OIXv2DDhg3Mnj2bq6++mp///Ods3LiRJUuW8KMf/Sh+vN/vZ8OGDdx99918/vOfH9W69+3bx8yZM+OXZ8yYwb59+7j++uv56U9/yjXXXMP3vvc9brnlFr73ve9hMOgfG53haNPIdnf7cFlNWE1TG/c+2MjtEUzte3VYegQHcueeTKbQEJA1NDQaBgYG6O/v5/TTTwfgmmuu4fLLL4/ffuWVVwJw2mmnMTg4SH9/P0VFReN6zlmzZrFq1SoAdu7cSWtrKwsWLOCzn/0soVCIW265hblz547rOXQOHzQp6pYeP1VTXBd/MFJVYKPXFyIQjmIzyxO/Tx1KM9U5An1rd5iQXiYphOC8887jqKOO4tprr81yL8n06dPZu3dv/HJrayvTp09POeb73/8+t956K3feeSfXXnstt99+e4pHoqNTqOYIQtGYPqs4A5px7EwaZK/NItBDQzpZcbvdeDweAAoLCykuLuaNN94A4P/+7//i3gHAww8/DMCbb75JYWEhhYWFvPDCC2zYsIF777035/N89KMf5c9//jOKovDuu+9SWFhIdXV1/PbXXnuNadOmMWfOHPx+PwaDAYPBoFcO6aSQnBMo0fMDw6iKN5UNxa/zadPJ9GSxTjY+9alP8cUvfpE777yTRx99lAceeCCeLK6vr+dPf/pT/FibzcbRRx9NOBzmvvvuy/h4d955J7fffjvt7e0sXbqUCy+8kHvvvZcLL7yQ5557joaGBhwOR8rjKorCrbfeGjc01113HVdddRWRSITf//73k/sG6BxSmI0G3FYTnmBEn1WcAa27OLmEND6dzKLnCA4bvF7vsOtWrlw5rFxTo7a2ls2bN2d9vJNPPnlYH8G7776b8djPfOYz/OY3v8m5vq9//et8/etfH3a9EIK77ror432EELz44ovxywsWLIj3GejopFPoMKuGQPcI0sk0xP5gKR/VQ0M6OjoThiYroecIhuO2mXFZTSkegT90cOQIdI/gMECr7NHRmWq0PEGpbggyUllgTfMIoliMBiymqd2T6x6Bjo7OhBH3CHTBuYykj6z0BSM4priHAHRDoKOjM4FoHoEuL5GZqgJ7ikfgC039mErQDYGOjs4EUqTnCHJSVWil0xMkGpMjKw8GCWrQDYGOjs4EUlPiwG42UuHWy0czUVVoJxpT6PbKpjJfcOolqEE3BBNKa2srl1xyCXPmzGH27NnccMMNhEKhjMfu37+fT3ziEyM+5oUXXkh/f/+Y1vPDH/6QX/7ylxmvnz59OkcddRSLFy/m6aefHtPjTzQ//elPp3oJOuPk0qOn89q3V+K26eWjmahKKyH1HgTTyUA3BBOGoih8/OMf59JLL6WxsZEdO3bg9Xr5/ve/P+zYSCTCtGnThs0fyMRzzz03bs2gTHzzm99kw4YN/P3vf+fzn/88sVgsr/tFo9EJX4vGWAzBZK5HZ/QYDYIKt64zlI30kZX+gyRHMPUrmAz++R1o3zSxj1m1BC64LevNr7zyCjabjX/7t38D5JCaO+64g7q6On70ox/xyCOP8Pjjj+P1eolGozzwwANcdNFFbN68Gb/fz+c+9zk2b97MvHnz2L9/P3fddRfLly+ntraWtWvX4vV6ueCCCzjllFN4++23mT59Ok899RR2u50//vGP3HPPPYRCIRoaGvi///u/vOcFLFiwAJPJRHd3N9dddx179+4lEAhwww03cN111wHgcrn40pe+xEsvvcRdd93FK6+8wjPPPMPQ0BAnnXQSf/jDHxBCsHLlSo4++mjeeOMNfD4ff/7zn/nZz37Gpk2buOKKK7j11lsB+Mtf/sKdd95JKBTi+OOP5+677+b73/9+XL110aJFPPjggxmPMxqNw9bz7LPP8vTTT2MymTj33HMzekE6OgcDWlOZpkIqQ0NTfxrWPYIJYsuWLRx77LEp1xUUFDBr1ix27twJwPr163n00Ud57bXXUo67++67KS4u5sMPP+SWW25h3bp1GZ+jsbGR66+/ni1btlBUVMRjjz0GwMc//nHee+89PvjgAxYsWMD//u//5r3u1atXYzAYKC8v57777mPdunWsXbuWO++8k56eHgB8Ph/HH388H3zwAaeccgpf+9rXeO+999i8eTNDQ0MpyqoWi4W1a9fy5S9/mUsuuYS77rqLzZs3c//999PT08PWrVt5+OGHeeutt9iwYQNGo5EHH3yQ2267La7e+uCDD2Y9Ln09CxYs4IknnmDLli1s3LiRm266Ke/XrqNzoCl1WjAbRdwj8AYjUy5BDYerR5Bj5z6VnHPOOZSUlAy7/s033+SGG24AYPHixSxdujTj/evq6jjqqKMAOPbYY2lubgZg8+bN3HTTTfT39+P1ejnvvPNGXMsdd9zBX/7yF9xuNw8//DBCCO68806eeOIJAPbu3UtjYyOlpaUYjUYuu+yy+H1fffVVbr/9dvx+P729vSxatIiLL74YkAJ2AEuWLGHRokVx8br6+nr27t3Lm2++ybp161ixYgUgZzhUVFQMW9/LL7+c9bjk9RQWFmKz2fjCF77ARRddxEUXXTTia9fRmSoMauhM8wj8oQiOg8AjmPoVHCYsXLhwWMx/cHCQPXv20NDQwPr163E6neN6Dqs1UYlhNBoZGpIqhp/73Od48sknWbZsGffff39encbf/OY3+c///M/45VWrVvHSSy/xzjvv4HA4WLlyJYGA/LDabDaMRrlrCQQCfPWrX2Xt2rXMnDmTH/7wh/HjktdoMBhS1mswGIhEIiiKwjXXXMPPfvaznOvLdVzyekwmE2vWrOHll1/m0Ucf5Xe/+x2vvPLKiK9fR2eqqC600TYwRDASJRxV9PLRw4mzzjoLv9/Pn//8Z0AmMb/1rW/xuc99bsR4/cknn8wjjzwCwIcffsimTaPLb3g8HqqrqwmHw/HwyWgZGBiguLgYh8PBtm3bsorbaSf9srIyvF5vXgnvZM466yweffRROjs7Aejt7aWlpQUAs9lMOBwe8bhkvF4vAwMDXHjhhdxxxx188MEHo1qPjs6BprLQRsdgMC5B7Zxi5VEYpyEQQpQIIV4UQjSqv4uzHHeNekyjEOKapOstQoh7hBA7hBDbhBCXZbr/oYAQgieeeIK///3vzJkzh7lz52Kz2fKqhPnqV79KV1cXCxcu5KabbmLRokUUFhbm/dy33HILxx9/PCeffDLz588f0/rPP/98IpEICxYs4Dvf+Q4nnHBCxuOKior44he/yOLFiznvvPPioZt8WbhwIbfeeivnnnsuS5cu5ZxzzqGtrQ2QEtdLly7lqquuynlcMh6Ph4suuoilS5dyyimn8Otf/3r0L15H5wBSXSA9grgE9UHgEQhFUcZ+ZyFuB3oVRblNCPEdoFhRlBvTjikB1gLLAQVYBxyrKEqfEOJHgFFRlJuEEAagRFGU7pGed/ny5cratWtTrtu6dSsLFiwY82uZSqLRKOFwGJvNxq5duzj77LPZvn07FovenTkRHMqfDZ3Dj3vfaOLWf2zl4etO4Ip73uXuq47hwiXVI99xAhBCrFMUZXn69eM1RZcAK9W/HwBWATemHXMe8KKiKL3qQl4Ezgf+BnwemA+gKEoMGNEIHI74/X7OOOMMwuEwiqJw991360ZAR+cwRZtUtqvLB0y9BDWM3xBUKoqi+evtQGWGY6YDe5MutwLThRBF6uVbhBArgV3A1xRF6Rjnmg453G436R6Ojo7O4YnWXbyrSw6yOhjKR0fMEQghXhJCbM7wc0nycYqMMY0mzmQCZgBvK4pyDPAOkLUTSAhxnRBirRBibVdX1yieRkdHR+fgQWsq29kpDYHjUOgsVhTl7Gy3CSE6hBDViqK0CSGqgc4Mh+0jET4CefJfBfQAfuBx9fq/A1/IsY57gHtA5ghGWreOjo7OwUjlMI9g6g3BeMtHnwa0KqBrgKcyHPMCcK4QolitKjoXeEH1IJ4hYSTOAj7McH8dHR2dwwaLyUCZy8K+ftkHdDDkCMZrCG4DzhFCNAJnq5cRQiwXQtwLoCaJbwHeU39+rCWOkYnlHwohNgKfBb41zvXo6OjoHPRUFdrQCjYPeRlqRVF6FEU5S1GUOYqinK2d4BVFWasoyrVJx92nKEqD+vOnpOtbFEU5TVGUperj7BnPeqYal8uVcvn+++/na1/72oQ/T39/P3ffffeEPV5PTw9nnHEGLpdr2HrXrVvHkiVLaGho4Otf/zpaufGNN97I0qVLufrqq+PH/uUvf+E3v/nNhK1LR+dwRUsYmwwCi3Hq+3qnfgU6IxKJRFIuT7QhsNls3HLLLRlVO7/yla/wxz/+kcbGRhobG3n++ecZGBhg/fr1bNy4EYvFwqZNmxgaGuJPf/oT119//YStS0fncEUrIXVaTQghpng1h6nW0M/X/Jxtvdsm9DHnl8znxuPSWyTy55lnnuHWW28lFApRWlrKgw8+SGVlJWvWrOGGG24gEAhgt9v505/+xLx587j//vtTZKuTFUu/853vsGvXLo466ijOOeccbr/9dr797W/zz3/+EyEEN910E1dccQWrVq3i5ptvxu12s3PnTs444wzuvvtuDIZU++90OjnllFPiKqkabW1tDA4OxruMr776ap588klOOeWUeM+D3+/HbDbzy1/+kn//93/HbNYHkujojITmERwMiWI4TA3BVKHp6Wv09vbG1ThPOeUU3n33XYQQ3Hvvvdx+++386le/Yv78+bzxxhuYTCZeeuklvve978XlpbVdd7pi6W233cbmzZvZsGEDAI899hgbNmzggw8+oLu7mxUrVnDaaacBsGbNGj788ENqamo4//zzefzxx/OajAawb98+ZsyYEb88Y8YM9u3bh9vt5sILL+Too4/mrLPOorCwkNWrV/Pf//3fY33rdHSOKKoK7QA4DgKdIThMDcF4du7jQdPT17j//vvjjWKtra1cccUVtLW1EQqFqKurA6TY2zXXXENjYyNCiLjoGmSXrU7nzTff5Morr8RoNFJZWcnpp5/Oe++9R0FBAccddxz19fUAXHnllbz55pt5G4JcfPvb3+bb3/42ANdeey0//vGPuffee/nXv/7F0qVL9bkA/7+9uw+OqrziOP79gUCsgFRAhEQFASkMkCAvWoE2MJURtWI60je10kpbnUJ9hYozTFHbqSLVdqqMdKwTsGoDKgg6I1iRgq0gKNGkWIuAbaNUKFWKIojk9I/7bNgEQpYsm032ns8Mk3uffe7mPOTunr3P3Xuuc0eROCJoDt8YAj9H0GSmTp3KlClTqKioYN68eTVVPGfOnMmYMWOorKxk2bJltUo6p1u2Gjhs/jFRHK+oqIiioqKjXtGcn59PVVVVzXpVVRX5+fm1+mzcuBEzo1+/fixatIiFCxeyZcsWNm/enHbszuWqxDmC5jI15ImgiezevbvmTXT+/PlHbC8tLU3puTp06MCePXtq1kePHk1ZWRkHDx5k586drF69mhEjRgDR1NC2bduorq6mrKyMUaNGUVJSQnl5OeXl5Qwbdlj9qRrdu3enY8eOrF27FjNjwYIFTJhQ64JyZs6cyZ133smBAwdq7h/cqlUr9u7dm9JYnIujQyeLm8fUkCeCJjJr1iwmTpzI0KFD6dKlS0379OnTmTFjBkOGDDns20H16dy5MyNHjmTgwIFMmzaNkpISBg8eTGFhIWPHjmX27NmcdtppAAwfPpwpU6bQv39/evXqRUlJyRGfs2fPntx0002UlpZSUFDApk3RtX1z585l8uTJ9OnTh969ezN+/PiabZYsWcKwYcPo0aMHnTp1oqioiEGDBrFv3z4KCwsb+1/lXM5r3+4EOrQ7oVncuB7SLEOdLblWhjpTVq1axZw5c2rdUziOfN9wzdFj6/5J327tGd6z4fOAx0umylA755xrhG+fe0a2Q6jhiSCHFRcXU1xcnO0wnHPNXE6dI2iJ01wus3yfcK5hOZMI8vLy2LVrl7/wXQ0zY9euXeTl5WU7FOeatZyZGiooKKCqqgq/aY1LlpeXV+vqaOfc4XImEbRp06bmal3nnHOpy5mpIeecc43jicA552LOE4FzzsVci7yyWNJO4B+N3LwL8J/jGE5L4eOOFx93vKQ67jPNrGvdxhaZCNIhacORLrHOdT7uePFxx0u64/apIeecizlPBM45F3NxTAS/zXYAWeLjjhcfd7ykNe7YnSNwzjlXWxyPCJxzziXxROCcczEXm0Qg6UJJb0l6W9Kt2Y4nkyQ9LGmHpMqktlMkPS9pc/j5+WzGmAmSTpf0oqRNkv4q6frQntNjl5Qn6RVJr4dx3x7ae0laF/b5Mkltsx1rJkhqLWmjpGfCes6PW9I7kioklUvaENoavZ/HIhFIag08AIwHBgDfkjQgu1FlVClwYZ22W4EXzKwv8EJYzzWfATeb2QDgPOBH4e+c62PfD4w1s0KgCLhQ0nnA3cB9ZtYH+AC4JnshZtT1wJtJ63EZ9xgzK0q6fqDR+3ksEgEwAnjbzLaa2afAH4AJWY4pY8xsNfDfOs0TgPlheT5wWVPG1BTMbLuZvRaW9xC9OeST42O3yEdhtU34Z8BY4InQnnPjBpBUAFwMPBTWRQzGXY9G7+dxSQT5wL+S1qtCW5x0M7PtYfnfQLdsBpNpknoCQ4B1xGDsYXqkHNgBPA9sAT40s89Cl1zd538FTAeqw3pn4jFuA1ZIelXSD0Jbo/fznLkfgUudmZmknP3esKT2wJPADWb2v+hDYiRXx25mB4EiSZ2AxcAXshtR5km6BNhhZq9KKs5yOE1tlJm9K+lU4HlJf0t+8Fj387gcEbwLnJ60XhDa4uR9Sd0Bws8dWY4nIyS1IUoCj5rZU6E5FmMHMLMPgReBLwKdJCU+7OXiPj8SuFTSO0TTvWOBX5P748bM3g0/dxAl/hGksZ/HJRGsB/qGbxO0Bb4JLM1yTE1tKXB1WL4aeDqLsWREmB/+HfCmmd2b9FBOj11S13AkgKQTgQuIzo+8CFweuuXcuM1shpkVmFlPotf0SjO7ghwft6STJHVILAPjgErS2M9jc2WxpIuI5hNbAw+b2c+zG1HmSHocKCYqTfs+8FNgCbAQOIOohPfXzazuCeUWTdIoYA1QwaE549uIzhPk7NglDSY6Odia6MPdQjO7Q9JZRJ+UTwE2Alea2f7sRZo5YWroFjO7JNfHHca3OKyeADxmZj+X1JlG7uexSQTOOeeOLC5TQ8455+rhicA552LOE4FzzsWcJwLnnIs5TwTOOddMSZoYCglWS0rpnsSSDoZidOWSUvqavCcC16wl7dSVkhZJ+lwziKlY0vlpPkcPSU803LPWNpMk3R+Wr5X0nXRicM1L2K9K6zRXAl8DVh/DU30SitEVmdmlqWzgicA1d4mdeiDwKXBtKhslXVmaCcXAMSWCuvGY2Xtmdnl9/RtiZg+a2YLGbu9aBjN708zeqtseakvdI2m9pDck/TCd3+OJwLUka4A+kr4a6s1vlPRHSd0AJM2S9IikPwOPSOopaY2k18K/80O/Ykl/kvS0pK2S7pJ0haKa/hWSeod+XSU9GV5s6yWNDMXsrgVuDEcqo4/U70jxJA8kxFYZlidJekrSc4pqyc9O6vddSX+X9ApRSQWSnvuWsNwn/D+8HsaZiH9a0htF4h4FJ0l6NvStlPSNjPylXKZdA+w2s+HAcOD7knqFx/IkbZC0VtJlqTyZF51zLUL4RD0eeA54CTgvFNaaTFR98ubQdQBRQa5PwjTSBWa2T1Jf4HEgMc9aCPQnKte9FXjIzEYoupnNVOAGoro195nZS5LOAJabWX9JDwIfmdmcENtjdfuF564VTwNDLCKqlrofeEvSb4jur3A7MBTYTVQ6YeMRtn0UuMvMFkvKA1pJGgf0JapBI2CppC8BXYH3zOziEPvJDcTlMkzSOqAd0B44RVEVWYCfmNnyejYbBwyWlDiqPJno770NODMUpDsLWCmpwsy2HC0GTwSuuTsx6YWxhqiWUD+gTFFhrbZEO3/C0qQ33TbA/ZKKgIPA2Un91idK9kraAqwI7RXAmLD8FWCADlUv7aiosmldR+u3NIUkANENRXaHeDYBZxKVCFllZjtDe1mdMaCo5ky+mS0GMLN9oX0c0ZtFInG0J3qjWAP8UtLdwDNmtiaF2FwGmdm5UFMmY5KZTUphMwFTj5QokgrSbZW0iugDhicC16J9YmZFyQ3h0/K9ZrY0vHhmJT38cdLyjUS1lgqJpkH3JT2WXHumOmm9mkOvi1ZERx7J25H0hk8K/T6u27keyfEcJP3XpoBfmNm8wx6QzgEuAn4m6QUzuyPN3+Wa3nLgOkkrzeyApLOJqqy2Bfaa2X5JXYimE2cf7YnAzxG4lulkDpUWvrqBftvNrBq4iqgo27FYQTRNBEA4sgDYA3RIoV+61gFfltRZUXntiXU7hDuxVSXmgiW1C1Niy4HvJY5MJOVLOlVSD6I3it8D9wDnHKdYXQZIKpFURVRW/FlJiSOAh4BNwGvhXNM8og8P/YENkl4nmkq8y8w2NfR7/IjAtUSzgEWSPgBWAr3q6TcXeFLR1yyfI/VP5wk/Bh6Q9AbRa2U10YniZcATkiYQJYD6+qXFzLZLmgW8DHwIlNfT9SpgnqQ7gAPARDNbIak/8HI4MvkIuBLoA9wjqTr0vS7dON3xYWargFV12hZzqNJocns1UWXd2+o89Bdg0LH+bq8+6pxzMedTQ845F3OeCJxzLuY8ETjnXMx5InDOuZjzROCcczHnicA552LOE4FzzsXc/wFyDQF3PGAMwgAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(np.arange(100000,100050,1), reverse_top10fft[100000:100050], label = \"FFT top-10%\")\n",
+ "plt.plot(np.arange(100000,100050,1), conc2[100000:100050], label = \"Original Parameters\")\n",
+ "plt.plot(np.arange(100000,100050,1), reverse_top10[100000:100050], label = \"Haar top-10%\")\n",
+ "plt.title('Parameter Values') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.legend()\n",
+ "plt.draw()\n",
+ "plt.savefig(\"ParametersWaveletHaar.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 175,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar tensor(12.0875)\n"
+ ]
+ }
+ ],
+ "source": [
+ "wavelet = 'haar'\n",
+ "coeff = pywt.wavedec(conc5000.numpy(), wavelet, level = None)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 176,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar tensor(9.4642)\n"
+ ]
+ }
+ ],
+ "source": [
+ "wavelet = 'haar'\n",
+ "coeff = pywt.wavedec(conc5000.numpy(), wavelet, level = None)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.2*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 177,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "6\n",
+ "haar tensor(12.1470)\n",
+ "(52814,)\n",
+ "(52814,)\n",
+ "(105628,)\n",
+ "(211256,)\n",
+ "(422512,)\n",
+ "(845023,)\n",
+ "haar tensor(16.9818)\n",
+ "(52814,)\n",
+ "(52814,)\n",
+ "(105628,)\n",
+ "(211256,)\n",
+ "(422512,)\n",
+ "(845023,)\n",
+ "haar tensor(2.3811e-06)\n"
+ ]
+ }
+ ],
+ "source": [
+ "wavelet = 'haar'\n",
+ "coeff = pywt.wavedec(conc5000.numpy(), wavelet, level = 5)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "print(len(coeff))\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))\n",
+ "\n",
+ "reduced = []\n",
+ "for i, o in enumerate(coeff):\n",
+ " print(o.shape) \n",
+ " if i > 3:\n",
+ " reduced.append(np.zeros_like(o))\n",
+ " continue\n",
+ " reduced.append(o)\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(reduced, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))\n",
+ "\n",
+ "reduced = []\n",
+ "for i, o in enumerate(coeff):\n",
+ " print(o.shape) \n",
+ " if i > 5:\n",
+ " reduced.append(np.zeros_like(o))\n",
+ " continue\n",
+ " reduced.append(o)\n",
+ "reverse_top10 = torch.from_numpy(pywt.waverec(reduced, wavelet = wavelet))\n",
+ "print(wavelet, torch.norm(conc5000 - reverse_top10, 2))\n",
+ " \n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# with resnet"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 178,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using cache found in /home/jeffrey/.cache/torch/hub/pytorch_vision_v0.10.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "model = torch.hub.load('pytorch/vision:v0.10.0', 'resnet18', pretrained=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 179,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "resw = {}\n",
+ "for k,v in model.state_dict().items():\n",
+ " resw[k] = v.clone()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flatr = []\n",
+ "for v in resw.values():\n",
+ " flatr.append(v.flatten())\n",
+ "concr = torch.cat(flatr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 190,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "bior1.1 43.02796936035156 115077.1953125\n",
+ "bior1.3 44.042415618896484 117252.8203125\n",
+ "bior1.5 44.798824310302734 119106.71875\n",
+ "bior2.2 44.7282600402832 117956.5\n",
+ "bior2.4 44.04926300048828 116528.5078125\n",
+ "bior2.6 44.09769821166992 116693.171875\n",
+ "bior2.8 44.207454681396484 116984.125\n",
+ "bior3.1 65.93879699707031 171188.890625\n",
+ "bior3.3 52.62727355957031 136826.25\n",
+ "bior3.5 50.10884475708008 130465.40625\n",
+ "bior3.7 49.333717346191406 128473.21875\n",
+ "bior3.9 49.07342529296875 127796.1640625\n",
+ "bior4.4 42.742881774902344 114508.9921875\n",
+ "bior5.5 43.98085403442383 118580.4296875\n",
+ "bior6.8 42.57365417480469 113812.046875\n",
+ "coif1 42.74231719970703 114690.046875\n",
+ "coif2 42.56312561035156 114001.890625\n",
+ "coif3 42.54220199584961 113903.578125\n",
+ "coif4 42.558956146240234 113906.859375\n",
+ "coif5 42.56120681762695 113924.28125\n",
+ "coif6 42.592472076416016 113980.0390625\n",
+ "coif7 42.60406494140625 114030.09375\n",
+ "coif8 42.61618423461914 114052.2734375\n",
+ "coif9 42.6173095703125 114053.078125\n",
+ "coif10 42.63934326171875 114081.7890625\n",
+ "coif11 42.65342712402344 114141.3671875\n",
+ "coif12 42.65858840942383 114128.3984375\n",
+ "coif13 42.66160583496094 114154.7890625\n",
+ "coif14 42.68099594116211 114187.8828125\n",
+ "coif15 42.693885803222656 114239.90625\n",
+ "coif16 42.69415283203125 114222.0234375\n",
+ "coif17 42.69487762451172 114230.65625\n",
+ "db1 43.02796936035156 115077.1953125\n",
+ "db2 42.69436264038086 114436.171875\n",
+ "db3 42.692832946777344 114409.2578125\n",
+ "db4 42.69171905517578 114270.2265625\n",
+ "db5 42.7407341003418 114356.6796875\n",
+ "db6 42.832889556884766 114583.046875\n",
+ "db7 42.90106201171875 114756.140625\n",
+ "db8 42.927757263183594 114787.8671875\n",
+ "db9 42.980587005615234 114925.3125\n",
+ "db10 43.0425910949707 115035.796875\n",
+ "db11 43.09166717529297 115145.9609375\n",
+ "db12 43.11075210571289 115177.953125\n",
+ "db13 43.153038024902344 115282.9765625\n",
+ "db14 43.23004913330078 115438.109375\n",
+ "db15 43.254371643066406 115495.4375\n",
+ "db16 43.26611328125 115499.40625\n",
+ "db17 43.29021453857422 115553.8359375\n",
+ "db18 43.339332580566406 115670.8515625\n",
+ "db19 43.363834381103516 115699.546875\n",
+ "db20 43.3875732421875 115747.3046875\n",
+ "db21 43.406944274902344 115809.53125\n",
+ "db22 43.44538879394531 115883.546875\n",
+ "db23 43.48051071166992 115981.046875\n",
+ "db24 43.502601623535156 115997.3203125\n",
+ "db25 43.519954681396484 116035.3359375\n",
+ "db26 43.53356170654297 116078.15625\n",
+ "db27 43.55718994140625 116114.859375\n",
+ "db28 43.56884765625 116135.7421875\n",
+ "db29 43.60223388671875 116207.0390625\n",
+ "db30 43.6269645690918 116254.625\n",
+ "db31 43.62778091430664 116261.8984375\n",
+ "db32 43.65283966064453 116317.5546875\n",
+ "db33 43.683868408203125 116371.8046875\n",
+ "db34 43.71052551269531 116451.03125\n",
+ "db35 43.69470977783203 116395.71875\n",
+ "db36 43.722896575927734 116448.71875\n",
+ "db37 43.732479095458984 116494.3203125\n",
+ "db38 43.75498962402344 116516.2890625\n",
+ "dmey 42.70671844482422 114251.421875\n",
+ "haar 43.02796936035156 115077.1953125\n",
+ "rbio1.1 43.02796936035156 115077.1953125\n",
+ "rbio1.3 42.904876708984375 114998.609375\n",
+ "rbio1.5 43.487056732177734 116603.4375\n",
+ "rbio2.2 48.12295150756836 128830.65625\n",
+ "rbio2.4 45.82650375366211 123338.625\n",
+ "rbio2.6 45.57454299926758 122649.625\n",
+ "rbio2.8 45.610652923583984 122736.25\n",
+ "rbio3.1 323.1489562988281 493834.4375\n",
+ "rbio3.3 63.0237922668457 168654.40625\n",
+ "rbio3.5 55.14375686645508 147856.1875\n",
+ "rbio3.7 53.33525466918945 143081.296875\n",
+ "rbio3.9 52.70606994628906 141449.8125\n",
+ "rbio4.4 43.57114028930664 116553.6171875\n",
+ "rbio5.5 43.684425354003906 115561.5078125\n",
+ "rbio6.8 43.01945495605469 115265.328125\n",
+ "sym2 42.69436264038086 114436.171875\n",
+ "sym3 42.692832946777344 114409.2578125\n",
+ "sym4 42.546844482421875 113956.859375\n",
+ "sym5 42.57182693481445 113959.1640625\n",
+ "sym6 42.52703094482422 113885.0546875\n",
+ "sym7 42.550376892089844 113925.90625\n",
+ "sym8 42.56036376953125 113907.7109375\n",
+ "sym9 42.602020263671875 114008.0625\n",
+ "sym10 42.568580627441406 113940.015625\n",
+ "sym11 42.623958587646484 114091.0546875\n",
+ "sym12 42.602752685546875 114007.203125\n",
+ "sym13 42.62556076049805 114069.9375\n",
+ "sym14 42.608367919921875 114038.21875\n",
+ "sym15 42.66935348510742 114179.7578125\n",
+ "sym16 42.629066467285156 114064.859375\n",
+ "sym17 42.639869689941406 114103.421875\n",
+ "sym18 42.6433219909668 114109.6328125\n",
+ "sym19 42.72231674194336 114301.625\n",
+ "sym20 42.658966064453125 114131.046875\n",
+ "min: tensor(42.5270) sym6 91\n"
+ ]
+ }
+ ],
+ "source": [
+ "# working on the random initialization\n",
+ "#wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " for v in resw.values():\n",
+ " flat = v.flatten()\n",
+ " #print(flat.shape)\n",
+ " lens.append(len(flat))\n",
+ " to_cat.append(flat)\n",
+ " flat = torch.cat(to_cat, dim=0)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " fft_layers.append(reverse_top10)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 181,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "11699132"
+ ]
+ },
+ "execution_count": 181,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(concr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 197,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " concr.abs(), round(0.1*len(concr)), dim=0, sorted=False\n",
+ " )\n",
+ "top10_og = torch.zeros(len(concr))\n",
+ "top10_og[topk_og.indices] = concr[topk_og.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 198,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(47.3773)"
+ ]
+ },
+ "execution_count": 198,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(top10_og - concr, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 199,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "to_cat = []\n",
+ "for v in resw.values():\n",
+ " flat = v.flatten()\n",
+ " to_cat.append(flat)\n",
+ "flat = torch.cat(to_cat, dim=0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 200,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(71.3879)\n"
+ ]
+ }
+ ],
+ "source": [
+ "flat_fft = fft.rfft(flat)\n",
+ "topk = torch.topk(\n",
+ " flat_fft.abs(), round(0.1*len(flat_fft)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ "top10[topk.indices] = flat_fft[topk.indices]\n",
+ "reverse_top10fft = fft.irfft(top10)\n",
+ "fft_layers.append(reverse_top10fft)\n",
+ "err = torch.norm(reverse_top10fft - flat, 2)\n",
+ "print(err)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 201,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor(42.6944)\n"
+ ]
+ }
+ ],
+ "source": [
+ "coeff = pywt.wavedec(flat.numpy(), \"sym2\", level = None)\n",
+ "array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ "#print(coeff_slices) # should be static so we do not need to send them\n",
+ "topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ "top10 = torch.zeros(len(array))\n",
+ "top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ "og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ "reverse_top10wv = torch.from_numpy(pywt.waverec(og, wavelet = \"sym2\"))\n",
+ "err = torch.norm(reverse_top10wv - flat, 2)\n",
+ "print(err)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 202,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACDNklEQVR4nO2dd3gc1dWH37NNWmnVi2VL7r3buGMDNsWmmBoIECAmJBACBJIvJIGEFGogkFACJBB6LzbFEKoBg8Hdxr3JVZZsq7eVVmV37/fHzK5X0qpZkmVZ930ePbt7587MHWk1Z+655/yOKKXQaDQaTffF0tkD0Gg0Gk3nog2BRqPRdHO0IdBoNJpujjYEGo1G083RhkCj0Wi6OdoQaDQaTTdHGwKNposjIjNFJLuzx6HpumhDoOlURGSviHhExC0iuSLygoi4OntcTSEi/UREiYitnY4XKSIlInJqmG0Pi8j89jiPRtMY2hBojgXOVUq5gBOAicAdrdlZDLrMd7m+AVFKVQFvAj+u188KXA68ePRGp+mOdJl/Hs3xj1IqB/gYGCUiCSLyoYjki0ix+T4j0FdEFovIvSLyHVAJDBCRn4jIVhEpF5HdIvLzkP4zRSRbRH4nInkiclBELhCRs0Vkh4gUicgfQvpbROQ2EdklIoUi8paIJJqbvzFfS8yZzDRzn2vM8xeLyKci0jfkeEpEbhSRTCAzzOW/CPxARKJC2uZg/I9+3NS11cc816CQzy+IyD0hn+eKyDpzFrJURMaEbPu9iOSY59kuIqc1dh7N8YM2BJpjBhHpDZwNfI/x3Xwe6Av0ATzA4/V2uQq4DogB9gF5wFwgFvgJ8LCInBDSPw2IBNKBPwP/Ba4EJgAnAX8Skf5m318CFwCnAL2AYuAJc9vJ5mu8UsqllFomIucDfwAuAlKAJcDr9cZ7ATAFGFH/2pVSS4GD5v6h1/eaUsrbgmtrESIyHngO+DmQBDwFLBSRCBEZCtwETFJKxWAYor2tPYemC6KU0j/6p9N+MG40bqAE42b+JOAM028cUBzyeTFwVzPHfg+4xXw/E8OYWM3PMYACpoT0XwNcYL7fCpwWsq0nUAvYgH7mvraQ7R8DPw35bMGYqfQ1Pyvg1GbGewfwmfk+1tx/fAuvLTtkmwIGhXx+AbjHfP9v4O56x9qOYfAGYRic0wF7Z3839M/R+9EzAs2xwAVKqXilVF+l1A1KKY+IRInIUyKyT0TKMNwx8abfPMD+0IOIyFkistx085RgzC6SQ7oUKqV85nuP+Zobst0DBBaq+wLvmu6TEgzD4AN6NHINfYFHQ/oXAYIx+wg73jC8DMwSkV7AxcAupdT3Lby2ltIX+E1gnOaxegO9lFI7gV8BfwXyROQNcyya4xxtCDTHKr8BhmI8scdy2B0jIX2C0rkiEgEsAB4Ceiil4oGP6vVvDfuBs0wDFfiJVMY6RjjJ3v3Az+v1dyrD5dNgvOFQSu3DcCldieEWevEIr60SCF1rSKs3znvrjTNKKfW6OYbXlFIzMAyGAh5oasya4wNtCDTHKjEYT+gl5iLtX5rp7wAigHzAKyJnAbPbcP7/APcGFnxFJMVcB8A8hx8YUK//7SIy0uwfJyKXHMF5X8Tw008HXjXbWntt64AfiYhVRM7EcPsE+C9wvYhMMaOtokXkHBGJEZGhInKqaXiqMH7//iO4Bk0XQxsCzbHKI4ATKACWA5801VkpVQ7cDLyFsbD7I2BhG87/qLn/ZyJSbo5hinmuSuBe4DvTvTJVKfUuxtPzG6YraxNw1hGcdwGQCHyhlDp4hNd2C3AuxrrLFRjrCZjHWg1ci7HwXgzsBK42N0cA92P8zg8BqcDtR3ANmi6GKKUL02g0Gk13Rs8INBqNppujDYFGo9F0c7Qh0Gg0mm6ONgQajUbTzWkX9cSjTXJysurXr19nD0Oj0Wi6FGvWrClQSqXUb++ShqBfv36sXr26s4eh0Wg0XQoR2ReuXbuGNBqNppujDYFGo9F0c7Qh0Gg0mm6ONgQajUbTzdGGQKPRaLo52hBoNBpNN0cbAo1Go+nmaEOg0WhaT2kObP+4s0ehaSe0IdBoNK1n1TPwxhXg13Vrjge0IdBoNK2nqhSUD2rKO3skmnZAGwKNRtN6qk0DUFXWuePQtAvtYghE5EwR2S4iO0XktjDbTxaRtSLiFZGL623zicg686ctpQU1Gs3RosZtvFZrQ3A80GbRORGxAk8AZwDZwCoRWaiU2hLSLQujLuqtYQ7hUUqNa+s4NBrNUUTPCI4r2mNGMBnYqZTarZSqAd4Azg/toJTaq5TaAOiVJY3mOGBPTQkLXNF6RnCc0B6GIB3YH/I522xrKZEislpElovIBY11EpHrzH6r8/Pzj3CoGo2mPXiXcv6akoTPU9LZQ9G0A8fCYnFfpdRE4EfAIyIyMFwnpdTTSqmJSqmJKSkN6ipoNJqjSLm/FoAKT0Enj0TTHrSHIcgBeod8zjDbWoRSKsd83Q0sBsa3w5g0Gk0H4lZe47WysJNHomkP2sMQrAIGi0h/EXEAlwEtiv4RkQQRiTDfJwPTgS1N76XRaDoVvw+3udxXXl3cyYPRtAdtNgRKKS9wE/ApsBV4Sym1WUTuEpHzAERkkohkA5cAT4nIZnP34cBqEVkPfAXcXy/aSKPRHGvUuHFbjFtHRVVJ545F0y60S81ipdRHwEf12v4c8n4Vhsuo/n5LgdHtMQaNRnOUqHbjtggA7hodNXQ8cCwsFms0mq5EdXlwRlAeSCzTdGm0IdBoNK2jxk2FmK4hb2UnD0bTHmhDoNFoWoW/qiToGirXhuC4QBsCjUbTKjyeYpSYawS+6k4ejaY90IZAo9G0ivLKw0lkbn9NJ45E015oQ6DRaFpFRXVJ8L1beXVxmuMAbQg0Gk2rKK86nETmFjksSa3psmhDoNFoWkWFqThqE4sRRqoVSLs82hBoNJpWUV5r1CLoYY81DIGuSdDl0YZAo9G0igrTFZQWmUS5nhEcF2hDoNFoWoXbzB1Ic6ZQYRE9IzgO0IZAo9G0CrfPA0BqVA/cFguqqrSTR6RpK9oQaDSaVuH2VhONhVhnEl4Rqjy6JkFXRxsCjUbTKtz+GlxiJcaZBECFNgRdHm0INBpNq3ArLy6xE+1MBKDco4vTdHW0IdBoNK3CrXy4rBHEOGKMzyGZxpquiTYEGo2m5fj9uMWPyxqJy+ECdHGa4wFtCDQaTcupceMWCy6bE5c9YAi0xERXRxsCjUbTcsx6xS579OEZQa02BF0dbQg0Gk3LqS7HbRFcdtfhGYHX08mD0rQVbQg0Gk2LqfWUUGWx4IqIJdoeDYDbV9XJo9K0lXYxBCJypohsF5GdInJbmO0ni8haEfGKyMX1ts0TkUzzZ157jEej0XQMFZX5ALgi4rBZbDixUq6rlHV52mwIRMQKPAGcBYwALheREfW6ZQFXA6/V2zcR+AswBZgM/EVEEto6Jo1G0zG4q4zkMVdEPAAxFjsVygtKdeKoNG2lPWYEk4GdSqndSqka4A3g/NAOSqm9SqkNQP1SRnOAz5VSRUqpYuBz4Mx2GJNGo+kA3J4iAFxmMlm0NYJyi0BNRWcOS9NG2sMQpAP7Qz5nm23tuq+IXCciq0VkdX5+/hENVKPRtA13VQkALmcyADE2J26LaCnqLk6XWSxWSj2tlJqolJqYkpLS2cPRaDofnxdqKo/qKd3VhtKoK8owBC5bNBWii9N0ddrDEOQAvUM+Z5htHb2vRtO9WfIPeOrko3rKQBZx0DXkcOniNMcB7WEIVgGDRaS/iDiAy4CFLdz3U2C2iCSYi8SzzTaNRtMM+bnrWe/eB77ao3bOQBaxy9QZinHEGK4hPSPo0rTZECilvMBNGDfwrcBbSqnNInKXiJwHICKTRCQbuAR4SkQ2m/sWAXdjGJNVwF1mm0ajaYanK3dzQ48UqCg4aud01xqLwoFkMldEvFnAXhen6crY2uMgSqmPgI/qtf055P0qDLdPuH2fA55rj3FoNF2a71+BvtMhsX+Luuf6KimzW6kqyyYytmcHD87A7fNgUxBhjQDAFRmPx2LB6ylpn5uJplPoMovFnUL5IVh4M1SXd/ZINMc73hp4/0ZY3fJnokJluISKind31Kga4PZV4cKCiADgChanOXqzEk37ow1BU3z7MKx9EfZ809kj0RzvBOr+lh9qWf9qNwXGvZjCsqyOGVMY3L4aosUa/BwwBOUe7dHtymhD0BieElj7svE+d0unDkXTDQgYAnfLDIFy51JgNW7IReUHOmpUDahQtcTIYSdQQIG0Qhen6dJoQ9AYa16A2gpwxEDe5s4ejeZ4p6qU5ZERFLkPtqi7u3QfNRZjSlDoyevIkdWhXHmJtjiCnwOLxuVVerG4K6MNQTi8NbDiKeh/svGTqw2BpmPxegr5RVoqr6iW3VALQtYFCo9i8fgK5SfGGhn8HCxXWavX0boy2hCEY8t7UH4Apv0SeoyAwl1Qq6V2NR1HeUUuXhFyaVm2cEHZYWWWoqNVKtLvp1wg2nbYEASlqLXWUJdGG4L6KAVL/wXJQ2HQ6dBjJCgfFGzv7JFpjmPKKwz9rDybtUXrBIUVhgvJAhR6j5LMRG0FFRbBZd78IWRG4NWGoCujDUF99i6BQxtg2g1gsUDqSKNdu4eOHyqL4MP/g+pjp8RiueneybdaWxQ5VFBprAv0sUZR5D869QBUVXmwTGWAYJUyXZymS6MNQX2WPQFRyTDmMrYXbeeHy+8g3+7UhuB4YtuHsPpZOLC2s0cSpKyqGIA8q61FhqCwqgSbgv6ORArFb6xrdTDVnkK8IrgcscG2CGsENgS3Lk7TpdGGIJT8HbDjE5h8Ldgjeez7x9hatJV1qf0gT4eQHjdkrzZej7JyZ1OUmxIN5VYLntL9zfSGgtpyEsVGcmQiRVYrVHS8NLvbnIWEGgIRwWWx4/bX6uI0XRhtCEJZ/gRYI2DiT9lUsIlvso1Esj0xyXpGcDyRs8Z4rT12/NrlIQu++aV7m+1f4K8i2RpJYlQyxRYL3vKWhZ22BXelWZ0sMr5Ou8tiFqepPXYMq6Z1aEMQoKIA1r8BYy8DVwr/Xv9v4iPiSYpMYrfDDu5cqDh6YXqaDqLaDXlb2Gm3H1szghCjlNdcLoHfT6HykmyPISm6J0qEkpJ9HTxCcJvrGDGmBHUAly2SCouuSdCV0YYgwKpnwVsF024MzgbmjZzH0MSh7AksxunEsq7Pge9ZFmHnwoyebCvv+JtnSykLifzJ9+Q23dlTTIHVQnJEAomxRjmPwhbMItqK21zHiDZlJQK4bFHGjEBrcnVZtCEAI0dg1X9h8GxIGRqcDVw+7HL6x/VnT1U+CrR76HggZzWrnEYcfEHVsaOPU+6rwoaRKZxnloNsDH/5AYqsVpKikkmK7wdAobvjZSYC1clinHUrBLrsLtyii9N0ZbQhANg031hsm3YTG/M3BmcD0fZo+sf2x+OrItel1wmOC7JXs8EVD4DnWFoj8NfQwxJBJBbym4nJLynZg0+EpOieJLl6AVBU2cwsoj3GaK5jREcn12l3OVyma0jLTHRVtCEA2PoBxPeF/ifXmQ0A9I8ztOH3JPfTkUNdHaXwZa9mo8148vYcQ4ub5cpLrCWCFFs0ec1kFxeU7AUgObYPiZGGv/5oyExU1BiunxhnPUMQEWe6hvSMoKuiDUFtlSEzPXg2Gws2sSRnSXA2AJASadTT2ROTAnlbwe/vzNFq2kJZDjtrCqnE+Bt6jlZGbnN4qykXRazNSWpEvJFU1kR2cUG5EV6aHN+fWEcsNnV0ZCYCC9pRjug67TERCVRYLCiPnhF0VbQhyFpqhL0NPqPBbODjjQc5/cHvcVqj2eNwGP2K93TygDVHTPZq1kVEBD96vMdINmxVGeUWCzH2aFKdyYbMRHnjrp7CCmNbclwfRIREsVN4FCQeKryVOBXYLfY67dHOBHwieI6hNRdN69CGIHMRWCPYGJNYZzZQWePlrg+34PNDJGnsUYHIIe0e6rJkr2K900liRAIAnmNFFqGqlDKLhRh7DCmuXuRbraiyxhd/C8xqYMmmiybJ6qToKFyL21tFtJIG7TFmFJFbF6fpsmhDsPNz6DeDf29+vs5s4N+Ld3GwtIpBqS5KyxLYU5UPiC5S05XJWcP6KBfjUscTocBzrMgiVJUaM4KIWFJj++KxWHCXNZ5dXFhTilMJUbYoABIdLlNmomOvx+2vxiUNbxmBTONAeKmm69G9DUHxXijYwYaMMXVmA1mFlTz1zW4uGNeLX58+BE9FEnmefCoS+0Hups4eteZI8NVSdGg9WeJjbOpYnGKh0lfb2aMCoLayEI/FQkxkAilxfYCms4sLaitItNiDdYOTIhKOisyE21+DSxqWqA9UKXPrqKEuS7sYAhE5U0S2i8hOEbktzPYIEXnT3L5CRPqZ7f1ExCMi68yf/7THeFpM5udUifCXopUkRiYGZwN3/28LNotw+9nDOW14KnZ/DwD2JvfXrqGuSu5mNlgNLZyxKWNxYsWjjg1D4DZDP2MiE0mJSgWazi4u8FeTbI0Kfk5yJlNotaJaWu/4SMepfLhCqpMFCCqQ1uqooa5Kmw2BiFiBJ4CzgBHA5SIyol63nwLFSqlBwMPAAyHbdimlxpk/17d1PK1i5yIe6Nmbne793DfjPqLt0Xy9I5/Pt+Tyy1MH0yM2kki7lel9jMvZ6UqCot3HlDSBpoXkrGZ9pAObWBmZNBKnWPH4jw1DEKhFEBuVTKppCPIbKz/praZQ/CSHCL8lRadRK0J5C8Tq2oJb+XBZIxq0B2YE5TU6s7ir0h4zgsnATqXUbqVUDfAGcH69PucDL5rv5wOnSWBe21nUVvHJoeXMj4BrRl3D9PTp1Hj93PnBZvolRXHNjH7BrpeMHYdSFlbVCig/5G/rvHFrjoxsY31gaOIwIm2ROC32YBhpZxOoRRATnUqKmbWb11gxeHceBVYryZEJwaaAzERRB8tMuEXhCilTGSAwI6g4VsJxNa2mPQxBOhD6KJJttoXto5TyAqVAQLCkv4h8LyJfi8hJjZ1ERK4TkdUisjo/v+2+0OztC7kzwcUYVx9uGn8TAC8t28vu/Ar+fO4IImzWYN+ZQ3si3iRWu80nniNwDyml8Pu1TG9n4c1eySaHjbEpYwFwWux4lK+TR2UQqEUQ40wmyh5FjNjIayQctLbsACVWK0lRPYJtSXF9ASh053TcIJXCLYauUH2CMwKvp+POr+lQOnux+CDQRyk1Hvg/4DURiQ3XUSn1tFJqolJqYkpKSrguLabWX8vv1v8LAf4+61HsFjt55VU8siiTWUNTOHVYjzr97VYLPZx9yK4uQNmcjUcOKQXv3wjfPNhg050fbOGSp5a1adyaI8RTTGb5Pjyow4bA6sCDOiY09AO1CGIijK9+ii2afHxQ2/DGWlRiFK1PiukVbEty9TS2VTTiTmoH/NXlVNSrThYg2mbWLT5WorA0raY9DEEO0Dvkc4bZFraPiNiAOKBQKVWtlCoEUEqtAXYBQ9phTE3yr7X/YqO3hL9ae5GeOAiAv3+ynWqvjz+fOzLsPuPShoC9gEJX/8Yjhza8Cd+/Al/eA8v/HWxWSvHJpkOs2VdMfrn+Zznq5KxlvZlINjY1YAgi8Ih0eMhlSwho+MSafv+UiHjyGilZWVBqKKYmx/YNth2WmSjosDFWVBqz8MDTfyhWi5UoseH21xwThlXTetrDEKwCBotIfxFxAJcBC+v1WQjMM99fDHyplFIikmIuNiMiA4DBwO52GFOjfJvzLc9vfp4flpUze8hFAKzbX8L8Ndn8dMYA+ic3fOIBOLH3cMTi4xt/j/CuIU8JfHYHpE+AYXPhk9thy/sA7C2s5FCZkfCzfLeuaXDUyV7N+sgIkiOT6BVtPEk7rU48x0gxlTJTuiFQCD7VmdJo7eLCcuMZKzlhQLAtwVwvKKrpuPDNiopAdbI4ALw+P8UVh8tjuiwO3IIh5a7pcrTZEJg+/5uAT4GtwFtKqc0icpeInGd2exZIEpGdGC6gQIjpycAGEVmHsYh8vVKqw9IT8yvz+eO3f2RwRBK/LSoxZKeB/36zm4QoOzedOqjRfQfEG+Jzi6qdRry2u940/Kt7obIQzvkn/OAZ6D0ZFlwL+5aybJdx87dZhGXaEBx9clazPsrF2NRxwdj7KJvTmBHUdL4Cabm3EgsEE8RSXL3Is1lRYaqOFZihpsnRacE2m8VGPFYKa90dN0ZzRhAdaRiCx7/ayan/WEytz1hwj7FG4tbFabosDbNDjgCl1EfAR/Xa/hzyvgq4JMx+C4AF7TGG5lBKccd3d+DxenhI9SQyvh8kDaS4oobPt+RyxdQ+uCIa/3UEVEh32uxQjSFJ7TJC/TiwDlY9AxN/Cr3GGW2XvwHPngGvX87utEdJjXExslcsy3dpQ3BUUYrCA2vYnxrNJeb6AIDTHoXHYjk2ZgS+KmKs1qCRSo3ri1eEkpK9JNTrW2guLCfVKw5jyEx0XGZvRSCyyXRDfbLpEMWVtewtqGBwjxiibU7cAQXSmB5NHUpzDNLZi8VHDRHh2tHXcufkOxiwdwUMPgOA99flUOPzc8mE3k3uHxcRR2JkIu4YM+o14B7y++F/v4GoJDj1jsM7RCXClQtQVgfX7L2VOX1h2sAkdhdUkFump89HRN42o6RoayjazQZl3OzH1jMEtSJ4j4En2HJ/DbEhQm4pZhRQXllWg74FNaXEYCGiXjx/oj2aQvEZarodMcZAdbLIBA6Weth2yIig23LQ+P3F2KP1jKAL020MAcDEtImcJdHg9cAgwxC8vSabkb1iGdErbLBSHfrH9ccZW0mBisWTvcFo/P5lyFkNZ9wNzvhg39LqUjb7K3h96i95P9aLr+K3VFoXAXqd4Ej53bsXseCTG1u3U84a1kcYiWQjkg7nOTrN2HdPY/H6R5Fy5SUm5MaeaoaG5oepOlbg85BkaZjUlRSRQKHVCh0UOVRhVk2LiUrh6+2Hw7cDBsHliDGrlGmZia5Iu7iGuhSZn4MtEvrNYMuBMjYfKOOv59ZPhA7PgLgB7Cj6jG3+3gzL2oCzsggW/RX6nGgUvQf+tuJvfLDrA8prQ7Isk2KwqVpid/6H2Mj7WbarkPPH1U+10DRFrTuXTyMs+Cuy+UFrdsxezXpnFMPMRLIATnNh1lNVQkz7DrV1eKspF4ixOoNNgeziPE+9fBmlKPDXkGyr6xYCQ2aiyGoFdz7E92n3YQZCXKOjUvhqex494yKJc9rZZs4IXI5YPSPownSrGQEQVBvFEcXba/bjsFpafFPuH9ef8tpS9rj6EVO+Ez7/k1Ge75yHQITdJbt5bdtrjE0dy60Tb+WRmY8w3noXroP3c6NzAEV4mdA/+thfMK4qg7fmGXIaxwjZOSvxi1Dsa13Skjd7JZsdDsamjqvT7owwDUFnP8FWlVFuEWJD4vMD8tJ59cdWXUahVUh2xDc4TGJ0D9wWC9Vl2R0yzArzwSYiMpnvdhYyc2gKw3vGsvWgOSOIjD+8RqDpcnQvQ1C0Bwp3wqAzqPH6eX/dAU4fkUpCdEMhrXAEFoyrMnoTQY2RMzD1F9DDyD2Ynzkfm8XGPdPvYd7IeczqfSqb9riYNiCdjLh+AIzqWcq+wkoOlBzDWZg7F8GW92D18509kiBZuesAKGqNUFxtFTuKtuERVWd9ACDKjNmv7HRDYEpQOw7PSxxWBwliJ99bLwqoPJdCqzVoKEJJijEq6RWZeQbtTXmNG1GKLbmCu9rLzKGpDO8Zw6GyKooranBFJlJlsVDr0VLUXZHuZQh2Gj56Bp/Bl9tyKaqoaXaROJSAIbBmmIU4HCkw04iErfZV88GuD5jVe1YwomNHXjlFFTVMG5BERqKRJ9fTaSwALjuWo4d2LzZet7x/zCQIZRVnAlCEv+VjOrSR9XZDKqS+IXBGxgPgqem4kMsWETQEcXWaU+wu8uplF3tK9+O2WEiKTm1wmERTvrqwvINmBN5KXAq+ySzEbhWmD0pmWJphTLcdKsflNKKJKo6T4jSHSqvYlNN91ju6lyHI/AwS+kPSQN5enU1qTAQnDW74dNUYPaN7EmGNoMip2BYxmj/5rqXGakzpv9j3BSXVJVw85OJg/8DNftrAJDJSxgDgrd1HfJT92HYP7fkabE4o2QeHNnT2aADYZ+rolFgEX0v90AfWsj4ygpTIRHpG96yzyRlh3Hg9nayYWVtZYNYiiK/TnhIRT76tblJZYVBeouHDS5KZV1BU0XiJy7ZQ7vXgUrB4ez6T+iXiirAxrKcxi9l2qAyXKY9RfpwUp7nzg83Me24l6hh5EOpoupchOOlWmHMfeeVVLN6Rz0UnZGCztvxXYBEL/WL7sac8iwMXLuDdilF8stn4R52fOZ90VzpTe04N9l+2q5DeiU4yEqKISx2Jy+8npzyLKf0Tj90ZQfFe42f6zSDWYHZ0Z5NVbTxp+kUoLWlh3eiSLNZHRjI2dTz1xW6dZjaup7ZzE8rKzSifmHp5AanOVLOI/eEbe0FZoGh9vwbH6WiZCbevimgsbM8tZ+ZQQ+srxRVBUrSDrQfLiLEbRsF9DERhtRWfX/HdzgIKK2o4WNo9Qr27lyHoMwWGnc27a3Pw+RWXTMxo9SH6x/VnT+keZg5JpW9SFC8u3cve0r2sOrSKHwz+ARazlJ/fr1ixp4hpA4x/cIlKJN3rJ6cyl2kDksgp8bC/qPOTmRqw+2vjdeRFxqL6seAeUoosVU2EOYyiFhqCwrIcsm1WxqaMa7DN6TBmcm02BGUH4NDGI949kLEbU8/vn+LqRYHViq/ssGxXoRlOGtAZqvb6KHQbWkkBd2RhB92IK/w1RCrjuz1zqOGaEhGG94xl26Fyos3fp7uz11zagc0HSimr8gKw9WD3WPzuXoYAI8P47TXZnNAnnoEpDQW0mqN/XH9y3DnUqhqumtqXNfuKeer717GKlQsGXRDst+VgGaWeWqYNNJ/0RMiwRJJdU8q0gcY/fWe7hxauP8DYOz8jJ3Thes/X4EqDlKEw4jxjcT1va+cNEqgpy+Gg1cJIm+F+KC5vWQGW9ZVGv4DQXChOmxGu6WmrdPLHv4NXGyTNt5hALYLYen7/HnH98ItQWLI32BZ42k+OMp7I7/xgC1P/9gWPLsrESgRRWDpMZqLc78Xms5Ie72Rw6uH/m2FpMWw/VE6ULVClrJPXXNqBpSGz9S0HtCE4Llm3v4SdeW4umdjyReJQ+sf1x6/8ZJVlccnE3jjtis/2fcgpGaeQEnVYHjuQNDZtwOEnvQxHHNmqmsGp0SRGOzpVbsJT4+Pe/22h1FPLs0vMJ2y/35gRDDgFRGDYuYDA1voagkeXQOjouIShABQ2UcYxlO3Vhr96eOLwBtsCuj5tMgR+P+z9DsoPtj7j2SS0FkEoKab7Jz8kCqigqghRhshcqaeWd9Zmkxjt4OFFOzj3X98SJxEU+jrGleHGB7UWThmaUsfNNqxnLNVePyVuY1HefQxIdrSV73YWMDjVRb+kqGDm9PFOtzMEb63OJtJuYe6Yns13DkMgcmhP6R7inHamjj5IDWXM6XNBnX7LdhXSPzmatLjDSUzpUT2oFuPJbuqARJbtLuy0xahnluwmt6yaUemxvLEqi9LKWkM2o7IABsw0OsX0gD7TOn2dICvfWLAe3+dUoOW6+/m+ChLFXieRLEBgRlDZlhtnwQ4IRMkc4aypfi2CAIHs4ryKw0avoNZNglixWWy8uzabqlo/z86bxDM/nkippxZvpZUi8eOpaP8FcLfyY/HZmTmkbi2QYWnG2kBOkSE+5+7iVcqqvT5W7S1i+qBkM09CG4LjDk+Njw/XH+CsUT2JibQ3v0MY+pr+2T2lxlN0TdQy/LXx7N5/uFCI1+dn5Z4ipg6ouwAYyCXIKdzGtAFJHCytYl/h0f/HyS+v5j9f72LOyB48ePFYKmt8vLJi3+Gw0f6nHO484nzDQBRkHvVxBthXZJx7zIDZiFIUV7UgRNFbTYHykmQLLytutVhxKPB4a8JubxFZS3k7Jpp7khKO3BCYtQhC8wiA4OwyP2Txt8BfRZI1EqUUr67IYmxGHKPS4zh9RA8++7+TSYyIp9Bq4ep//Y+lu9p30bhCwOp3MH1Q3ZnL4B4urBYhK980BF28OM26rBKqav1MG5jEiJ6x7C2sxF3t7exhdTjdyhB8uvkQ5dVeLpnQ+kXiAE6bk17RvdhTtof95fvZULiKNDmZ15Zn4zUleTcfKKO82nt4fcAkI3EYAPvz1ge3NbZOUOgp5LrPruNgC90greGRRTuo9vr5/ZnDGN4zllOGpPD8d3vx7VoMSYMhLiTTevi5xmsnzgqyKnKIVZAYnUq8kpbp7lfkU2C1khIR32gXJxY8/rYYguW8HZfAOzEuanOPbMG43FysjnXUnREkRiZiAfJqSowGn5dC5SXZHsPqfcVk5rn50ZTDUhKxkXZGp2VQZLWSSAlXPbuSnXntMzOo9dZQbRFiHNFE11PojbBZGZgSTWZuFXYslB/B7zOvvIqq2laUDfUUG8mc2/4HB743JOH97VN/+rtdhVgEpg5ICuqPbT90/M8KupUheHvNfjISnA2e1FtLIHLoncx3sIiFa8dfxoHSKhZtNVwWgZv71AGJdfbr1WMMohTZxTsZmOIiJSaiUQG6lYdWsuzgMj7Z+0mbxlqfzNxy3li1nyun9mWAuVj+85MHUOquwL/3W2N9IJS4dMiY1KnrBPtqSugrhnsnQWwUtyTSx51Lgc1KcmTjf2unWPD4W5GpXP8U+5ax3W6hVoT9+Y1UrWuGcm8lVg67qgLYLDaSLBHkB1wtlQVGVnFEAq8u30dMhI1zx/aqs09idColFgv3zk5FMNyg7cHOXGPRvYcrPuz2YWmG1ITLYqcCf6uqvu0vquS0f3zd8ph9vw/euMIoCfvGj+DpmfDQYLi3Bzwyhqpnzw1b0KelLN1ZwOj0OOKcdob3NAxBd1gw7laG4E9zR3DvhaOxWKT5zk0QMATv7XyPk9JP4uKxI0mPd/Li0r2AsT4wKNVFakxd33RE0iBSfD5yyvcjIkwdkMSyXeHXCTLNTNqlB5a2aaz1uf/jbUQ5rNx82uBg27SBSVyUegi7z4O//ykNdxp+Hhxcb0h0HG2UIkvV0Me8oSdaIij0N+/XV2WHKLBa6xRwqY9TbHjUEU77S7NZV5NH4Dk0s2zfEYXZlvmqiMHaIM8BIMV2OLtYlR+iwGohxpHIR5sOceEJ6UQ56j6dJ8Vk4BdB1Rxg1rBU3lmbEywc0xa+3bYDgN4J4Y3q8J6x5JR4iLZEUN4K4blan5+b3/ieyhofK/YU8faaFhiu7x6Ffd/B2Q/BtV/Bpa/CWQ/C1F+QGzeGyP3fkPXl0y2+tlAqqr2s218SjOrrGRdJfJS9WywYdytDMCzNcIO0lf5x/fF4PRR4Crh4yMXYrBaunNqXZbsL2XyglFV7D+cP1CEyjgwfZJt+32kDksgrr2Z3QcMn3IAhWJu7lqp2Kv+3dGcBX2zL46ZZg0gM0VcSEa7plYVPCV9VD2244wiz0NzWD9plHK2humQfh6wW+ppaOol2F0UtuHmXle6jVoTkmMbdgE6LjcojNQT7lrEmMgKbWLEg7BQvlB6+kXl9fjJzm3fNlPtriLGEX69KjUgwahe7c3GXZlFtsVBUEUON11/HLRQgKDNRls0PJ/amwF3N4u35Dfq1lvV7jAeAXrHhs/ADGcYOMauUtVB47tFFmXyfVcLDl45jUr8E7vtoazAvIiwHvjcqAY64ACb9DNJPgOFzYcp1cMZd/MX2a1b5h+Dc9s4RGeWVe4vw+hXTB5m5PyIMT4tly8HOzT4/GnQrQ9BeBCKHUp2pzEifAcClk3rjsFm4bcFGKmt8DdYHAmRYnWR7jS9WcJ0gTBhpZkkmSZFJ1PhrWJu7ts1j9vsV9360lfR4J/NO7Ndg+5CK1eywDuTJ5WFcVQn9oOfYTlknyM5ZiRKhjxk6muCIpVgU+Jq+geebWbgp5gJ9OJwWOx51hE/MWUtZ64xiRNII+jhTyXTY6ywY3/XhFuY88g1ZzQQDlCsvsdaGUU1gyFEHZCYKTHmJTQcjmNA3IajzE0qSKV9dVHGQmUNTSHY5eHt1y3IuGqPG62d3npHU5qongxFguDkWiz+CCosYirzNsHRXAU8s3smlE3tz3the3HfhaCqqvdz30bZGBlJplH6NToW5DxvhzSEUVdTwxbZc3vdNJ8Wz54iS/JbtKsRhtTCx72GX7ohesWw/VIbPf3xLTWhDcAQMiBuAVaxcNOQibBZjep4Y7eD8sb3YaApVNbYOkRGZSJ7yUuOroV9SFD1iIxosGFfUVpDjzuGiwRdht9jbxT303rocNh8o43dnDiXSFGILUu1Gclbj7Xsya/YVs3pvmKicEecbBXhKO0bUrDH25a0HoE/aOACSIhMptVqpdTetqVNQYWThJrkaL5votDjwcGSGoHrfUjZGODihxwQGJw1jp8MerFq3am8RLy3bh1/B/zY2sdjvraasXi2CUFJi0im2WqkpzabAlJfeUxLPFWFmAxCSXewpwG61cNEJGXy5LY+Cpp6ym2H13iJEmTUHnOG/0z1iI4iPsuPzma6hZmYERRU1/PrNdfRPjuYv5xm1QAb3iOHnJw9kwdrs8BFPn/3RSG688D9G9b96vPd9DrU+hW/Y+dQqKxVrXm/llRr5A+P7xON0HP7/GNEzlqpaP3vCzNqPJ7QhOAKSnEm8OfdNrhtzXZ32wJP2sLSYOq6XUDKie6EEDrhzEBGmDUhiRb18goBbaMzyZzkhfgjLDi5r03iran08+Ol2xmbEce6YXg077FsKfi+Dp55DQpSd/3wdpg7B8PON16PsHsoq2QVAn7RJACSaT70lJU3XSsg3pRtSnI27Ap3WCKOAva/5BeON2aWHn+4ri9hYvodaFCeknsCgpBFk2WxU5W6iqtbHbQs2kB7vZHjPWD5qyhBUlRnKo/bwIa6pZsnKgpLdFJpicg5bKmePDp8Dk2SuoxSZ1cQumZCB16947/ucsP1bwlfb84iwGdcdUBitT8CFUlUT0WxxGqUUv5u/geKKWh67bHyddY6bTh1E36Qo7nh3U90oou0fw+rn4MSbGgYzmLy9JpvR6XH8aNYJfO0fg2XTAmNhuYUUV9Sw5WBZg/DY4ILxcb5O0C6GQETOFJHtIrJTRG4Lsz1CRN40t68QkX4h224327eLyJz2GM/RYGjiUOz1fLuj0uO4fHJvrpzat9H9MuIGAJCdZ0SZnDq8BwXuGj7dfDjSIbPEMASDi/YzraqGHcU7gje2Wp+f/3trHb9+cx3//WY332YWNPCrKqXIKqzkfxsOcv/H27js6eUcLK3iD2cPD79QvudrsEYQOWA6V03rx6KtuezMqycVkDwIUkfCllZGD/lqofLIpYn3VRwkXkGcWQY0wWXcBJvT3Q9o7oTT7g/gtEXisQjUNP20t2pvET/491IufPI7due7IWs5ayOMcpEn9DiBQfGDUCLsLtjME1/tZFd+BfddNJqLxqezMaeUfYWNHL+q1ChKY+YQfLktt07IZ4rpgswr28+hCuPvf+7IIQ1ndCaxjlhsCIVmEZnBPWIY2zuet1bvP6LExRqvn4XrD9AzzrihusLIXwcY1jOGkkqHWa6y8Zvmy8v3sWhrLredNYxR6XWltyPtVu4+fxS7Cyr492LjAQB3Hrx/E6SNhlP/FPaYm3JK2XqwjB9OzGBkr1gW2U7BWZVrLCq3kOW7C1GK4PpAgEGpLuxWOe4jh9psCETECjwBnAWMAC4Xkfq1H38KFCulBgEPAw+Y+44ALgNGAmcCT5rH67L87aIxTRuCZONXk11guBHOHpXGgJRo/vHZjqAfMrM4k2ix0svrY1qW4RpZfnA5AG+szOKdtTksyczn3o+2cuWzK5hwzyKm3LeIq59fyY/+u5yxd37GyQ9+xY2vreWZJbup9fm57axhTGksbHb3YkOQz+5k3rS+RNgs/PebME/cI86DrGVQ3gqp468fgMcnNevTb4ysmhL6WAw5iOW7C7HYDFdPUTOVuApqy4nEQnQjT9sATqvTmBE0IYuwr7CC615aTa94w49/1bMrqchcwlqnk0FxA4mLiGNQwiAAdlTk8PTiHVw0Pp1ThqRw1mgjYqlR91CwFkEsO/PKueaF1cx5ZAl3frCZUk8tqdFm7eKKg2RXFGFRMG9q42VVRYREi6OOzMQPJ2awI9fNhuzWi8F9uOEAuWXVpJsTAVdU44ZgeFosHm8UFRZBecKfa+vBMu7531ZOHZbKT6b3C9vn5CEpnD+uF/9evIudueVGmGiNGy56BmwNazUDzF+TjcNm4byx6VgsQvXAOVQQidrwVouv9btdBUQ7rIzJiDcaDm2EHZ/isFkYnBrTIRnGpZW1Rkb/MUB7zAgmAzuVUruVUjXAG8D59fqcD7xovp8PnCZGvNz5wBtKqWql1B5gp3m845bk1FFE+P3kmC4Pm9XCb84YSmaem4XrjSl8ZnEmg/wWxB7FsPICEmzRLDuwjLKqWh5elMmU/oms+uPprP3TGbz6syn88ezhTB+YzKHSKsqrvJwzxlh8W3jTdDbfNYf/3XwS158yMPyA3PmQuykoK5HkiuCSiRm8+30OeWX1opVGnA8o2NZy99C3mR/wlyg/6kjqGvh97KOWvpFJvL8uh8ueXs4Lq42EpaKKJmLFlSLfV0WyNTJsWGaAKHsUHrEYC5FhKPXUcs0Lq1DA8z+ZzIvXTKbUU8vudYv4PjKSCWkTAegT0weHWNllFUZGFvKnucbNOiMhivF94vnfhvCGoKaygCqLhZjIBF5etg+H1cLFJ2TwwtK9zHpoMUu2Gdea5ymkyFtBrN/KkB4NF4lDSbK5KBJ/cJZz7theRNgsvL2mdYvGSimeWbKHwakunBG12JUiIoxvPsCwnjHU+GLwi1AZJvO7qtbHL1//njinnQcvHtPk3+WOc0YQabew+NX7jRoiZ9wNqcPC9q32+nhvXQ6zR/QgLsqYoU8dmsEnvon4N78PtS2LuFu6s5DJ/ROxWy1GAatnZ8NrP4TtHzO8Z2y7u4Z257s59R+L+fkrq9v1uEdKexSvTwdCv2XZwJTG+iilvCJSCiSZ7cvr7XtcV3WXhL6ke31km5LCAGeNSmNEz1ge/jyTc0b3JLM4kzMqymHcj7Bsfo+pysHSA0t58qudFFXU8MdzhhtPf9FGyn99v2ar2PuN8dp/ZrDpZzMG8OqKLF5Zvo//mx0STpoyDBIHwI7PjPC95qgoZIGvgEUxLn6Q+QFj0k9o1dCqinaRa7MSbenBrW+vxxVh49u9dmKGQHH9wu6hVJdRaIFke9Nl6Z32aGosgq+mnPrT0FqfnxteXUNWUSWv/HQK/ZONmcUzPxqBf34WlZLKqKRxgJH8lWhJJdPh5vaJqk7p03NG9+Se/21lb0EF/ZLrzk7KTb9/hD2e+WuymTumJw9cPIYfn9iXOxdu4Z6Fe4kfBlmVRbittSRZmzYCAImOWAqtFsOlktif2Eg7Z45KY+G6A+YNtmUT7mW7CtlysIz7LxpNZlYFLr8f7FGN9h/SIwaljEVvt6eI+vOwx7/cyc48Ny9dM5kkV/gn+wApMRH85fQ0zvj8OXJTptJj8rUN+iilEBG+2JpHSWVtHRHJGYNTuN03nR/UfGsYkkD4cyMcLPWwu6DCCMndOB/evd40PALvXMe0E15iQXk1+eXVpMQ0PfZwKKWo8ddQ5a2i2lfN/pJSbnxtBcU+N6tza3hvezlWWzWVtZVUeCuoqK2gsraSSm9l8DXQ5vF6+M8Z/yHd1b63yfYwBEcFEbkOuA6gT5/wURNdAkc0GcpCdvXhSCGLRbh1zhCueWE1zy/fQGlNKYOrKiF9AoiVadve4OOkOJ7fvowLx59wePpan6oysFjB0bg7pAG7F0NEHPQaF2zqlxzNqUNTeW1lFjeeOogIm3nzEIGBp8G6V43s0Uam6gHUnq9ZH2HcFD/I+ZoxLR8VAPsPrARgxe4oBiS7+NePxnPmI4uxKkVRU5Ww3HnkW60MNIvPNEaU6TbyeIoIFSRXSvHn9zfx3c5CHrpkbB2X2lTHHl52Gk+eby6xc84APwdLqjhUnIqK2s9EZ92n/7NMQ/C/jQe5cdagOtvKK43omF2FNipqfPzYDDYY2SuON38+lQ82HOTvq+0U+8socVjpZa6TNEWSM4ndVmvQEAD8cGJv3l93gE83H+L8cS27gTzz7R6Soh1cMD6dv+w2qpPVD9kMJdJuJSUqjjLAXV1MaKzWjtxy/vP1Li46IZ2TW5jHc1HJS/iliqvyf8hDeW4G9zCMulKKh9c+zJvb3uS8geexaetoesZFMyPkYSg93snBxCmUVsYTt/GtZg3B0p3G/+LZ1R/Bgjug74lw+evUVhbhfvY0Rm39P1yR83hnq4W+qUJZdRkl1SUUVxUHX4uriymtLqXGV4PX7zV+lJdaXy3ecLkqKRBt/ir+tLzuJotYiLJFEWWPqvOaEpVClC0Kawd4z9vDEOQAoZrOGWZbuD7ZImID4oDCFu4LgFLqaeBpgIkTJ3bpoN4Mm4u1vorgUw3ArKGpnNAnnv+u+A6SYXBtrRG7nzKUaWuegaQ4LFE7uHXO5eEP6quFZ06DmDSY14rInt1fQ/+TDAMSwrwT+/Hj51by0caDXDg+JClr0Gmw6r/GWkFApbQRcnd+Rr7NRiTCJzV5/M5Xi91q3ETfWZvNE1/t5EdT+nLl1D6HjU0Im/evA6DGMoQXr5lMWlwks4amsd0nFNU0MVV351JgtTC5iYghAKe5SOupKqljCJ5ZsofXV+7nhpkDubi+LpWZSBZvS2XZVi+3vr2eoooaVG06ubb1lOduJPS5PT3eyQmme6iBITBrEXy728/YjDjG9Y4PbhMRzhvbize3xFFUW06h1cKwRqJ2QkmMSqXQYkW5cwnctqcNSCI93sn8NdktMgQ788r5clsevzp9MJF2K25fFa4WeJH7JCSyyQflIYvFfr/i9nc2EhNp445zGl/fqEPeVmTNc1SNnUfO5r789MXVvHfjdBKjHTyx7gme3/Q8Y5LHsCBzAbWWN0jvN54VB51M6zUt+P80fUga76+eylU7PkU8JRDGiHr9XrLKsnhv+2LOSn2X17Zs4MCAkRxIcHHgvXMpri6G1GjAiyQ8y+NbgRBtwWh7NPER8SRGJpIUmcTAuIE4rA5sFht2ix2bxRb8ibBGgN/GK8sPklvi48aZwxnVK4nb5m+nX0Ii9184CZfDRZQtCqfN2aTrrCNoD0OwChgsIv0xbuKXAT+q12chMA9YBlwMfKmUUiKyEHhNRP4J9AIGAyvbYUzHNOmRSbhrsymrKSPOrJ0rItw6ZyjzFiwkEhjit0DyULBYiXf1o1eNF5WeTXp8+Jhz1rxgyCIX7IA930D/k5sfSNEeoy7xtJsabJoxKJkBydG8sHQfF47PQCnFkpwlvJ21kIujXZyy84tmDcG6g8sgCq5JnsSTBSv5NvN9Zg79AY8syuTRLzJJjYng7g+38Px3e/jtnKGcO6ZXMKqpwF3N1zvXQzw8cNklQTnvyyf34cFvreSpxiN9qsuyKbNaSWlm+uyMMA1BSFWtzzYf4r6Pt3L26DRund0wy1rt+47vo6I5ue9UevUayoOfbgfg6tMnsSDnI3YWbae+A+ycMb24+8Mt7CmoCLqYAMpNX/q+Yjt/n9sv7Bh7RCaS6T5o6AxFNZ4TESApJoMai1BRlh00bhaLcPGEDB77MpOcEk/j3yGTZ7/di8Nm4Soz6MHtq8bVwHnWkIHJyWzKhbIQQ/DayizW7CvmoUvGNhpSXQel4NM/QEQM0bP/xH8nWLj06eVc/8oaTp2yiac2PMUPBv+AP0/7M//8ci1Pf/8qFRlr+fminzMgbgA/HPpDMlwZ9Ojh4VXrIKZYviBq3UvYx/yQ/eX72VG8g+1F29levJ1dJbuoDqilJsEq4ukVk0ivyERGJI8iJSqFWEcssVmriF33Optiz2LOZXcQ44ghITIBh7UF12NSVevj6udXsn9fKk9fNYHThht/y7MHpfLKin2kRGY0EPQ7mrT5zKbP/ybgU8AKPKeU2iwidwGrlVILgWeBl0VkJ1CEYSww+70FbAG8wI1KqVbIEHZNMlwZUJxNdtk+4lIOO0xOHJhMWkoReAVX0jCw2lBK8Z5vOqd6FvJW5FaqfdXG00UoVWWw+H7ocyIU7zHe9zupyak8ABvfNl7D3NAtFuHH0/ry1w828vjKN/ji4JvsLNkJgDelF6fs+hK4u/Fjl+xnQ20JEZLA1WOu5Y3Pl7Fw21ssXD+Id7/P4ZIJGdx74WhW7Cnkbx9t45Y31vH0N7u5/azhjO8TzzUvrCLNUkyCH8ZmHM59mDk0hSe+cVBE44agsDQLgOS4pl2IyvRkL962hxVb17Ixp5SsokrGZsTxj0vGNQy19XnZc2gtRWkJTOgxgQsHDcTrU+wrrODaKakseAd2VuVxQq0H7IdvtmePTuPuD7fwUT33UJlpgGIj4jinkfoYKVE9+NK+HZ8ISa4wOSD1SIo1JtiFZfvrzHIunpDBo19ksmBNdh2dqfoUuqt5Z202PzghPejLd6taelmav1UMS0nh/VworjLCV3PLqnjg422cODCJC8f35K3tb3Fan9OCiW9hyfwcdn0Jc/4G0UmMj4YHLx7DrZ8+ztb1H3JO/3P409Q/IQgffe9mnOuHvHLJA3y691Ne3vIy96+8//Cx+sHF9IQd/zV+TBIiEhiSOIRLB11I3x1LGZazksq0C5jy4/8g1jAGb9iPWLpxPzfmvgt5l8LQs5r9XYTi9fm56bXvWb67iEcuHRc0ApTm8CPHNzjU9+TN/5T+UVVQkW8UObI6YNDpMGSO4RkQIa+sitdX7ufH0/rWWYdqD9rFBCmlPgI+qtf255D3VUDYen5KqXuBe9tjHF2FjIRBULyc/fmbGZlS13MeG1tAWl41Wy39GQV8tiWXxwsncF/8W7wSZ8hNTOs1re4Blz5mFJSZ8zZkr4aPfwt7lzQ9KyjZD0v+achMpwxpsLmythJfzBJcg/7LU1tLGBQ/iPtm3Mea3DV8vPN9anN3YC8/ZLiiwrHnGzZERDAybhDOXhM5s7KGtyzbKN6xk9/OGccNMwciIpw0OIXpv0xm4foDPPjpdq58dgXJrgiKK2uI719N33r1BGxWC4kOFwf95eQUV5Ke0HABM98sZRmo7VufVXuL+O3b6ymr2gd9YNmu/WyKKGF0ehyXTe7N5ZP61MkuDXJoPWtsRibyCaknICLccrpxU1VKEW2JINNuM2ZlPQ+Xx+wZ52RC3wQ+rOceKjTDLM8fPaDRRdyUmHR8hwyDlBzXeFhygESzjkFRxSFCe/dOjGLagCTmr8nmplmDGhVefGV5FtVePz+d0T/YVqG8xFgaXygOMKpXGmyCMjMc984PNlPt83PvhaP57sB33L38brLLs/m/if8X/gC+WiODOHFgnWCEaud3RKZ9SG3ZKAbwU6wWK2v2FbG7oILrZxrumHMHnsvcAXPZX76f8ppyKr2V/OWD75lY8RmTvEvxnHEnPZOGMTRxKCnOFOTAWljwM/xFe3mw9of8cO4/whsBABFWj/wzsct2MfKda5Frv4Lkxo1pff7w7kYWbc3lzvNGcsF4c5ZaWwXPzWFQ6X5us0PNzgiITYXoZGPxoKoEFv8NFt8HMT1hyByWVozmP+sTOX9cr2PTEGhaR0bqKNgN2QV1i5l4/V4OVe7j1JoqFhQnk+au5v6PtxGZ0p8J8WOxqYMsO7C0riEoOwhLH4dRFxsiXKkj4Nt/wld/a3pW8OkfjNc599Vp3l++n7d3vM07me9QWl1KcuRQDmZdwFMX3khqbCQR1ggWZC5gc4SDcbu+hHH1vYAGNbu/YmuEgx/1msre4mqGVCTjjangytNKGvjKLRbhgvHpnDU6jZeX7eOFpXu5/4Lh/Gejj6kRDSOi+sQms6U8l3eXb+ems8Y32F5gyk8kRzXcd9XeIuY9t5LUmAgunDCMt/LhhxOT+fdZp4b/PYWybxlrIyNJikgIFigKICIMiu3LzopSQ3MoxBCAET1014db2J3vDsp/HygvgSi4akrDUpoBUs2SlQDJTegmBQjKTFQ2jKr64aQMfv3mel5YupefTO/XwA9dVevj5eV7mTU0hUGphyOuyvETXX8WGoYBicYaRqW/mi+25vLRxkPcOnsI/ZOjuf/zVwH4dO+n/HrCr8P7wFc/ZxjRy98Am3GjW7hrIfcsv4eT0k+mVq7i/o93MDg1jk83HyLKYeWckCxrEaFP7OFZ4FkD4lmwyMJdEZ9DRRWMPcmoW/Ddo/Dl3eBK4+H0f/J+QR9+l9x0gMXgjFSuq/k130T9FdvLFxoPWa4exoOQK9Wo8x3fp24tD4xkt7dWZ3P9KQPranyt+i+U7odLX+W29Ul8sqOc1Tefjs0ashZTUWBEPe34BLVxPhfUvMDZkQ4ctV9Aq0MvmkZLTHQCUUlDSPT5yCnbW6c9qyyLGn8tg2trWV3dh0ufWsaeggr+cPYwosf+iPFVVSzb92Xdgy2+D/xeOM3MurRHwoz/g6ylxlpBOHZ9adQXOOk3EN8Hn9/HN9nfcMOiGzjnnXN4afNLTOoxiZfOeoln57xAddkw3lxlPGVPMqUeVsUmwc4vwh9fKbbt/5YaEeKtg7no30sp9AynX20t+6u/bvT3EmGz8rOTBvDt709lbj8PeTZrnX/sAGkxKbgtFr76fkNYMbCCKiMap768xOq9RVz93ErSYiN56+fTuHjySAD8qoV1i7OWsTYqmhPSJoa9kQ1KHkmmw4E61LA2QSC5LCA5UVXro7imAquCAUmNRzcFsosBkqKaj7hJjDRuxoXVDaOqzhrVkxmDkrnrwy1c88Iqcuvliby/LocCdw3XnjQg2KaUogJFjK35GYHL4UIUVKla/vTeJob0cHHdyQPZXbKbpQeWMiRhCAcqDrCpIEzthsoi+Oo+ozrekDMB+CLrC/703Z+Y0nMKD8/6J49eNpFhabH88vXv+WD9Ac4e3bNJv/pJQ1LYq3pSnDAaNrxl1Cl45UJY9BcYejYrzlzIE7tTmT0yrdnF2RG9YjlAMovHPWLc/Hd9ZRiUj38Hb18Nz58JD4+EdXU1jp77dg/RDis3zArJ4/GUwJJ/GBF4w+cyc1Q/SiprWbW33t8sOtl40PrhSyw4fQlX1NxOwbCrjDDudkYbgs4grjcZtV6yK+qGGu4oMXTfB9f6GDRqErvyK5g+KIlZQ1NhxAWcWF3LNncWBYHyhXlbjUpNk68zFEIDnPBjiOllrBXUlxbw1sBHv4OE/rgn/YRnNz7LOe+ew41f3Mi2om1cP/Z6Pv3Bpzw862HGp45nYIqLkwYn88qKfdT6/CREJjAkYQgr4pINgxJOz6Ugkw1+Q6LizW8tOKwWzjv3Is4tr2Bt/jpy3M1r32TlGDEDfZMaPi0nmoumNZ79fLOj4ZNvQU0ZglHkPcBqcybQIzaS16+bSmpsJE4zfLSyGYkJAJTiYPYKDlhgQo8JYbsMThxKqdVCQV7DG13POCcTTfcQwP82HKRGanE1UosgQCC7GJqWywgQuOZwUVWRdisvXTOZv547gmW7C5n98De8vy4HpVQwgWx4z9g6yrkerwefCK4WGAKLWIjAhseiyC91c9+Fo3HYLLy69VUcFgf/nPlPbBYbn+37rOHOX//dkKaYcx+I4Fd+/rH6HwyOH8yjsx4lwhpBlMPGM/MmEmm3UlHja7bSYKDAzDcRM+HQBnhiCmStgHMfI++sp7nxnT30T47mt3PCSK/Xo3dCFK4IG0uqB8LPFsFvtsKfCuDWnXD9d3DlAug9GT7+PZQZOUJ5ZVV8sOEAl0zsTWxoadzvHjWqrJ3+FwBOGpyCw2bh8y3hM/aVUjy7LIeClBPp+cN/BGdL7Yk2BJ2BPZJ0bGQHyhCaZBZnYgUGxA/kV2eOZvqgJP48d6Rxo3DGMy3VuAEtzzE1VBb9FRwxcPKtDY7PSYFZQb0n8OVPogoz+WTS5Zz34Q95ZO0jpLvS+ccp/+DTiz/lhnE30CO6bnTK1Sf2I7esOqiHNDltMuv8FdR4iuDguobXt+drNkQ4iLclsOOAhdvPHkbaiJM4p8K44f5v9/+a/RVlmU+NfdIaJqElxhrT77ToYl5fmdVge763gkRxBJVh1+yrawR6xBoRSIGqYJ6WFFwvyGSNGanUqCGIN/zGAa2o+pwzpifbDpWzK9/NS8v2Umv1EddM5EmqKevQnFxGALvFTpzYKfSFn+VYLMLV0/vz0c0nMSAlmlveWMcNr67l3e9zyMxzc+1J/esYJneNsfDrasG5wVB0dVuEqyckMLFfIqXVpXywayHn9JpB35yNnBg3hE93fYiqKDz8kJK/w3CVnPBjSBsFwKpDq9hfvp+rR11NVEgiW694Jy//dDK3nzWMyf2bDqe1WoTpg5J4qmg8yuaE+N7w82/wjf8xN7+xDnd1Lf++ckKLonUsFmFYWkzdDGOLBVwpxpgHnQ4X/Bt8NfDBr0ApXl6+D69f1ZXTKDsIy/8Noy8Jug+jI2zMGJTM51sPhdWEWr67iK0Hy7hmRkN3XnuhDUEnkeGI5ZCvCq//cLJJZnEmfbx+InqOo29SNK/+bCpD0w77aoeNu5p4n49lO96DPUtgxydw0q/DyvIy/qqGs4LSHPZ89xDXDRjObzNfIdmZzKtnv8qzc55ldr/ZDUT0AswcmkqfxKhgBbbJaZOpVl7WR0bAzi8b7rB7MeudUbjL+zI63VQ8dcaTnjCECTj5YNcHzYqg7TPVRfukNvSFJpruopE9a/hiW15dKQy/j0JVS7L5BLtmXzHznltFaj0jAOC0BwxBC2QIspayNjICly0qeMOvT0BzaGdtaVhN/rNG9UQE/vbRVtZnl1Jt9RPTjO/dZXfhtEaSFN28+yJAki2aIlFQ7W60z4AUF/OvP5HfnzmML7bm8X9vradHbARz66nTus1cB5ej6SztAAn2KNwWC7dGfwwLfsY7L52Gx1fFFctfhTevYM6OJRysKmDjY8Pgnh7wyGh46TywOWHWHcHjzN8xn1hHLGf0PaPBOYb3jOXnpwxs0e9jxqAUtpRFsveKJfCzLyFlCI8s2sHy3UXcc8FohvRo2XWB4R7aerAcf2O1CZIGwml/hsxPqVn7Gq+uyOL04T3omxRiRL++33DlzvpjnV1nj+jB/iIP2w41LILz3Hd7SIiytzgZ8EjQhqCTyHCm4hXIrTw8Hcws2sbgKk+DhcYA1sFzmFrjZ1nBetTnd0BsOky5PvwJgrOCZbB7MR6vh8c+/AkX9YhnsxX+MOUPvH7O64xJaX7RyWqGkq7aW8zmA6VMSJuARSysSukLu+qtE/h9FGR9ywGr4C5L5/azhh2OUOkzhXNLCtlbtpfNhZubPGdW5SESleCKaPiPmmj6zdNiPPj8qm6Jw4oC8q0Wkh1xvL8uh3nPrSQlJoLXr61rBCBkRuBrgSHYt4w1UdGM63ECVkv46JLEyEQS7S6zNsHWBtvT4iKZ2DeBRVvzSIhQuAViG6lFEEBESIlKJbkF6wMBkiLiDJmJ8iYksDH+rr+YOZCFv5zO9EFJ/HbOMBy2urcEd4VRh9vlaF7eAiA2Mha3xULEyifxZi3ndYePSY5khs59HK79kpkXvIhNLHw2cg5M+Tn0ngrJQ+Cch4yna6CoqohFWYs4b+B5DUOlW8lJgw132uIDNrA5WLw9j399uZMfTsxomCzYDMN7xuKu9pJd3MSaUuCaPv49torcOtFXFGTC2pdh4jXBrO8Apw3vgQgN3ENZhZUs2prLFVP6tlge5EjQhqCTyIgxok6yTTnlytpKsisOGhnFaY3cnG0OpqWMJV/Vsit/M5x6R5149QaYawXfL76TC+efxX+9BznT1Z+FF33I5cMub/SGFo5LJvTGabfy4tK9xDpiGZ44nBVRLti/su7T78H1rMdI0hmbMoYTQ3WQek/hjNJCHBY7H+xqOvs5q7acvtbw7ohEs9hMtb+EqQMSeWNVVvAprbrkAAVWK1l5Vm55Yx0ZCU5ev3ZqMCEtFLvFjl2BJ5BU1ATFB9ey22Zp1C0UYHDcQLNa2Zaw2wNRLpePjmuyFkEoZ/Y/k9P7nN5svwCJUakUWazGjacFDEuL5dWfTQ17YwzOCJqR6wgQ7eqJO20k3JbFVxc9zEG8XDH9Dhh7GaRPIHbQGUxPP4lPa3JRZ9wFP/gvzFtobDdZuHMhXr+Xi4dc3KJzNkXvxCj6J0ezJLOAAyUefv3mOoalxXDneaNafawRwdoETSi5Wqyo8x9Heat5LOZFpvQL+b19cZfx/3rybxvslhITwfje8Q0MwQtL92IV4appzYcOtwVtCDqJjCQjdj873yipF0jWGlzjDfpJw3HimGsAeLdHXxhzadMnsUWQPeWn3CR5SEU+z5XD385/q0WLjvWJi7Jzwfh03l93gOKKGianTWZDbbFR4Ss0OmnPN2yIcICy8Kcz6k3re08h1q+Y6erPJ3s/odbfiASvt5os8dGnEYkIl92FTRk1By6f3If9RR4+3nSIx7/M5DfPf0qh1YrDkshTV03go5tPCmsEAjix4PHXNHv9a70lQOPrAwEGp4xil92BPze8ITh/XDpnj07jxyfEU24RYlrgcvnl+F9y9airm+0XICmuL4VWqxGK2VJ2fGZkmtcjYAiinS0zBDGOGNzKC5FxvLLlFdJd6czMmFmnz5x+czhUcYgNBQ0VaZVSzM+cbwQqxDeimNtKZgxKZvnuQm56bS01Xj9PXHFC+DyRZhiaFoNFaLaG8bfF8fy99hKm1q5ENs03GrNXG5F6J/4yOPOpz+yRaWzMKeVAiTHjKK+q5a3V+zlnTM8Gs9n2RhuCTqJHykhsSpFTZDy1BaqSDYnqBWHcIQHSBs3moqh+vGSvYeGephddPV4PvypahhILTx08xKQz/t70DKIZ5p3Yl2qvn3v+t5UTUifhVT6+d8XXCSOt2vElKyJiibP2Z2xGvS984gCISubcWitFVUUszQlfgrMyfyv5NmuDWP0AIkIiVopr3cwZmUZ8lJ0bX1vLQ5/tYGhCOV4RLp44njkj0xpNnArgFAseXzOGwO9njdTiwMLIpJFNdh0UPxiPRcjJC18zNyHawZNXTCDNUUO5xUJsRFzYfm0hMTqNcquFmvztLdvBVwtvXgmvX2aICYZQYYr7xTSVDRyCy+GivKacrYVbWZu3NuzMc2bvmdgtdj7b2zB6aHXuavaV7WuX2UCAkwYnU1njY21WCff/YAwDU1zN7xSGSLuVASmuZovUPPvtHj6IPB9/xiT46LdG/Y5Ff4WoZJh2Y6P7nTHCmOku2mrMCuavycZd7eUn0/s3uk97oQ1BJ2FN6E9Pr5fsMsM1lFmSiVNBeo/w6wNBRLjjoneYnDaZvyz9C6sOrQrbTSnFncvuZEdxJvePvoE+p/yh1anx9RmWFssNM426sv/4oAarWFmVOsBYJ1AKvNX49y9nR4SFWf0mhh07faYy/dAOEiIS+GB3ePdQ1gFDo71PUuMiZUkWB0U+D5F2K7efNYxLJmTw0c0nceYIMws3JBGrKZxixRNOHTKU6jLWRTgY7ezRrL5McMG4dFfD0N0QgrUIIuJbNM7WEEwqK2ykEHx9inaDrxrytxmFhEIoNyu9RbdwFumyu3DXuHll6ys4bU4uHHxhgz4xjhim95rOZ/s+w6/q1ox+e8fbxDhimN13dsvG3gKmDUzCFWHjx9P6cu7Y5mU6mmJEz9gmi9TszHOzeHs+V544AMsF/4Zaj7EYvncJnPK7Jh/yBqa4GJASzedbcvH5FS8s3csJfeLriBF2FNoQdBZxGWR4fWR7jMW4zIItDKquxtKrGUMA2K12/jnzn/SO6c2vvvoVe0v3Nujz6tZX+d/u/3HjuBs5eeINRvJYO/C7M4fx7ytOYG+eF5+nN18LUJIFhTvZvW4x++x+ai2KaY3VHug9GXvRHs7MOIWvsr4iu7xhpbF9BcZCcp+0hlnDARJsURSbrqVLJ/XhwUvGMqJXLPlmnYeUmJZFWDjFhqc5eStPMbk2K32djVfoCjAo3jAEmarakIJuhEAtgpY+abeGwExqd9m+Jo1RkMB6RvpE+PYROPB9cFOFKSDnim7ZYrXL7qLGX8PHez7m/IHnE9vIIvPsfrMN91D+YfdQSVUJi/Yt4twB5xJpaz9XSEykne9+fyp3ntf0bK4ljOgVS06Jh6e/2UWN199g+/Pf7cFhsxi1DZIHw6l/NAxsfF+Y8JNmjz97RBrLdhXy/roc9hVWcs2Mjp8NgDYEnYfVTrpEkFNbhlKKzOIdh6WnW0BcRBxPnPYENouNG7+4keIQff5Vh1bx0OqHmNV7FteOaVjUo62cNbonH/xyBrEMI9NbgFsEX+Yi1n+zkHVmLd+xqY1cR++pAFwW1Z8IWwSX/+9ylh1YVqdLVmkgdHR0o2NIdMQYIZL1SmAGpBVaug7itNippGlDoCoLKbFYiWvB03u0PZr0iCQzcqjxyKgyc5wxYWQw2sqQBGP9abvUNGmMguRtAwQue9WQS3jvRiPxECivKcfp92ONjG/RuV0Ow+1S66/lR8PDy4/AYffQp3s/DbYt3LWQWn8tPxjygxadqzXERdnbJQb/skm9mTU0hfs+2sbsh7/ms82HY/9LKmtYsDabC8elkxwovjPtJph6o5Fj0IJEsDNG9MDrV/z5/c30jItkzshGtLzaGW0IOpEMRzxFysv+8v0UeysYXNNExFAYesf05tFZj3Ko4hC/+upX1PhqOFRxiFu/vpXeMb25b8Z9WKRj/sT9kqO5/+yLQBQfRaSx+ov5ZJSsZLUrlaTIJHpFNzIF7zkWrA4GFOzmjXPeINmZzPWLrueFTS8E/6GyKnNJVk0nUCVEJFBktRhieyHkmwaxvrxEYzitDjzNPDVXVeRRYxHiWmhcBiUOMSOHGoaQBgjUIoipl7zXHsRFxNEzIoHtDkfLFozzthjhjDFpMPcRw4AteQiACm8FMX4/OFrmV3fZjX4z0mfQP67xp9kYRwzT06fz+b7P8St/cJF4TMqYoCE7FomPcvD8Tybzwk8mYbNauO7lNVzxzAq2HizjtZVZVNX6+cmMfod3sFjhzPug3/QWHX9873iSXRG4q738eFo/o3TmUUAbgk4kI9oIJfxq/1cADHYkhE8Oa4JxqeO4d8a9rM1byx3f3cGvv/o11b5qHj310eDTWUcxqed47BY7G3r2Y0ztBsZbdrEtOpIxKU3UpLVHQq/xsH8lfWL78OrZr3J6n9P5x5p/8LtvfkdlbSVZXjd9bE2PPTEqBY/FQmVp3Vq8BbXlOLHUyUZtiihrBB4hvFSGSanbyKiOa2Es/6CkEey126nNDaOpY1JuGqzYI4jgaglDE4aw3WFvmSHI32aIFQIMPRPGXGZo4RzcQHltJdF+1eKqd4FM6KtGXNVs39l9Z5NbmcuG/A2szVvLntI9XDy4/RaJO5KZQ1P5+JaTuOv8kWw5WMY5jy3h8S93MmNQMsPSWpZzEQ6LRZgzsgdRDiuXT+7d/A7thDYEnUiGqSb5VZaRnTu4icXRpjiz/5ncPP5mPt7zMZsKN3HvjHsZEDeg+R3bSKQtknGp49gRBU6pwW1RZPnczSep9Z5s+KFrq4iyR/HQKQ/x6wm/5rN9n3Hlh5ex0+KnbzNP9EnRxpS5uN76SIGvihRry/3LTmskHosEi72HI2gIXC2bpg9KGIRXJLjWEY5ysxZBTMSR3zSaYkjKWPba7VTnNz4rAYwoocJddYXMzvwbRCXhfv8XbKsuJF5J87UtTKb0nMJ757/Hib1ObLbvrN6zcFgcfLr3U+bvmI/L7mJOvzktOs+xgN1q4cfT+vH1rbO4+sT+KAU3zGx7yOvtZw/n41tOIj6q/TWFGkMbgk4kI8n45/s+73uSvD4Se4WJtGkhPxv9M34x9hf8ccofOa3Pae01xGaZlDaJbZUHKLU52OA0nsLHpjSzztF7qqHJcnA9YISDXjPqGv496iZyS3ZTZrXSp0fjC8UACTFG8lNR6IygpoIC8TdbtD4Upy0SjwjUNq43VOox/PlxLSgMAyGaQ2VZhuxxGMpMDZ+W5BEcCUMTh+ITYWcTxggwks6UD1JDxP2iEqk96+/8ijwO+iq5vrrlvnWLWFoc/+9yuJiePp2P93zMZ3s/45wB57R4JncsERdl58/njmDLXXPqJlAeIa4IW11ZiqOANgSdSGzSEGJ8fvwoBtfWtHihOBwiwg3jbuCyYZc137kdmZI2BYVidf/JrE8bikWaj7Wn92Tjdb9Ztdvvg68f5MQPfscbbjvn9TyJOWbiXGMkxhuRMcWhCq5m0frkiJYlPwE4bVGGIWhqRuAxSkrGtaBmMED/uP5YETKtfigIH8tfXmvoAHWkIQDYUd5QlK8OgXWMEEPgV37+VLScFc5I7iwo4kTpOBfj7H6zKawqpMZfwyVDwtau6jIc7TrD7YkuTNOJSEI/MrxetlodxkJxz/YtNnE0GJ08GqfNycqBJ7K7dA9DapKaf6pzpRrJZftXGsk271xrqKSOvoTecx/m3iZirQMkxpglGStDomLceUZt3xYuFAM47VFUWyz4atyNVuUtNWPp41qY/OWwOujrymBnRYUhDhj6tG1S7vVgs0JkK9xYraF3TG+cYmO7r9gQn4to5GaevxUsNkg6LKT36NpH+d/u/3HLqGs578vHoIVZxUfCzIyZOCwOhiYODRovzdFHG4LOJLYX6V4fWyNgsDgbL/t4DGO32hmfOp7lh1aSV5nH2f3PbtmOvafAto/gPzOguhzO+5ehmNrCp6pEM/6+yCwAD1BVlk251UKKK3z933BEmW6kKk8JjU3GS2sNN05LDQHAoKThbC3dbxi4Kdc12F7uqyLWaus4WWGxMCSqF9sq3VCYaSzQhyNvq1Ea0gxtfG3razy36TkuHXopPz3hl9B7Nqjw7q32wOVw8dApD9GrhW43TcegXUOdicVKhqk+OSRhUDOdj10mpU1id+lu3LUtWCgO0HsKVJcaUVLXfmkI5LXipui0OYlUUFx9OMuzoMTQykkKU9Ws0eOYkVWeMBW9ApTWVuBAWvX0PihhENlWqNz3bdh1gnJ/DTGNyH63F0MTh7LD4UDlNxE5lLcVUo21qkX7FnH/yvuZ1XsWt0++3TBSqcOgx5EFMbSUWX1m6dlAJ6MNQSczOTKVQTU1DEw78oXizmZK2pTg+2YXioMdLzeSbK798ohuNCJCgtgo8h7W3C8ws5RT4luejek0ffSeMPUDApT6PMRL657exySPQQHrqILchrpDZcrXbC2CtjI0bSLlVgsHc9eF71BTCcV7IXUEa3PX8vtvfs+YlDE8cPIDrVKm1XR92mQIRCRRRD4XkUzzNawzUUTmmX0yRWReSPtiEdkuIuvMn+Zz+I8zToofzrs5h4js1YgkQxdgeNJwou3RxDpiGxWKa4A90qjH2sL49HAkWiIpCpGQLqgwwjyTo1v+NXKa4ZuVYUo7Bij11xJrad1Ne3zqeGxiZWVkhLFOEIq3mnJRxDRTi6CtDEk2DOz2xiKHCrYDCl/yEG79+lZ6uXrx+KmPB+s0aLoPbZ0R3AZ8oZQaDHxhfq6DiCQCfwGmAJOBv9QzGFcopcaZPy3Ihz/OCNQabkPEUGdjs9g4f+D5zB0w96hGTiTaoynCG9TTKTCzdVsjs+00/f6e6kYMgd9PKT7iWnlzjLJHMTJ5FKtc8XVlugGqygzlUXvHJvwNSRiCANvdDfWcgGDE0CaHg3xPPjeOu5H4FkpJaI4v2rpYfD4w03z/IrAY+H29PnOAz5VSRQAi8jlwJvB6G899fHDCPEOQKqFjC090NLdPuf2onzPBEccOS46x2BwZS35NKRabIT/RUpyRhiGorGlEY766lBKL0MfW+pnL5LTJPJe/gYqspUT7vGA1/92qSo2iNB0UOhogyh5FH6uLHd58Q5PJWu/fPW8rWB18696DRSxM6zWtQ8ejOXZp64ygh1IqEMh9CAgnnJIOhOoAZJttAZ433UJ/kiYeJ0XkOhFZLSKr8/Pz2zjsYwhXCozp2vHTnUWSM5FiixVlCqsVeitIEker/NtOs/KWp7E8Ak8xZRYL8UdQN2BS2iR8KNZaauHgusMbqkqNojQdUIugPkNc6Wxz2KBkX8ONeVsheQjfHljK6OTRrYqK0hxfNGsIRGSRiGwK83N+aD9lKIa1QPO2DlcopUYDJ5k/jQqUKKWeVkpNVEpNTElpeZy45vglMSqVGotQUbYf/H7y/TXBovUtJbBG4PE2bghKLZYWKY/WZ1zqOGxiY1VkpBFGalJdmU+1xUJsC8s/toWhSSPYb7dTkduwGhj52yhKHsjmws3MSJ/R4WPRHLs0awiUUqcrpUaF+XkfyBWRngDmazgffw4Qqp6UYbahlAq8lgOvYawhaDQtIsEU7Ssq3QeeYgqsQlILi6wHCBawb0Riosqda9y0j6BugNPmZEzKGFbGxNdZJyg3C8LHRLZOYPBIGNrLiOjKPLCy7oaqMijdz9JoFwrFSekndfhYNMcubXUNLQQCUUDzgPfD9PkUmC0iCeYi8WzgUxGxiUgygIjYgblA43KNGk09EmON54ui8mxw51JgtZIS2bobdtAQeD1ht5eUG57PliqP1mdS2iS2Wv2U718ZLANZHqxF0PEz26E9jGi07fWrlZllLL/1l5MYmcjwpIbZz5ruQ1sNwf3AGSKSCZxufkZEJorIMwDmIvHdwCrz5y6zLQLDIGwA1mHMEv7bxvFouhGJZjnKIvch/OUHDXmJVur7R5muJI+vKuz2UrOSWHwrspVDmZw2GT+w1uY3CpgTWoug46Ol06LTiFUWtlfk1N2QtwU/sLR8Nyf2OrHD6lZougZtihpSShUCDaQulVKrgZ+FfH4OeK5enwpgQlvOr+neJJra90WefIpL9uETITmmdRrudqsdmwJPSD5CKGWtVB6tz9jUsTgsDlY6nZyy5xvoNz1Yi6AjqpPVR0QYao9je2W+EWYbiMfI38aWqBiKa8r0+oBGZxZrui4J5mJrcVUJBWVGVExyXOvDcJ0IHl9N2G0l5tN7S5VH6xNhjWBs6lhWxcQbBcw5XIugsXq+7c3QmD5k2iz4zLoKAORtYUliLwRpUe0AzfGNNgSaLkukLZIoJRTVllMQKFof2/qqTk6x4vHXht1Wat602xJaOSltEtssPkpzVkFNZYfXIqjPkOQReCwW9md9d7gxbxvfRtoYlTwqaFA13RdtCDRdmkSLnUJvBQXBovWtX4CNasoQHIHyaH0mp01GAWscVti/grIOrkVQn6HpxhP/9kOrjIbKIkoq89jkczM9vWW1dDXHN9oQaLo0iVYnxf4a8qsNF07SkYR5ig2PCl+z+EiUR+szOnk0EVYHq5xO2PMN5V4Pdgy30dFgYPpUrEqxvcgskpO/jWXOSPwovT6gAbQh0HRxEu0uivBTWF1GdCuK1ofitNjx0Igh8HmIa6XyaH0cVgfjUscH8wnKfVXE0HG1COoTYYukv7Kxo9IUAcjbwrfOSOLsMYxKGnVUxqA5ttGGQNOlSYxIoNhqId/rblXR+lCcFjuVjRRfKfXXEmdpexHxyWmT2SFeig+tp9znIbaDaxHUZ6gjge0+I3van7uV76KiODF9upab1gDaEGi6OAnOJIqtVvKtFpJsR6bm6bRG4BHVsIBMUHm07QXVJ6cZSfOrI2yUKx8xHVSisjGGxvThkFUoLcthW/4GCq0WZmTobGKNgTYEmi5NYnQPvCLssdtJOcLoF6c1Eo9YoL7MRHUppRYh7ggNTCgjk0fitEayMiraUB49ypr/Q83KcduzFvNthRFqq8NGNQG0IdB0aRJdhpBtcSuL1ofitDnxWKShIQgKzrU93t9usTO+xwmsdsVRZrEQcwSy1m1hSIYRHbQ96xu+syuGRyS3qm6D5vhGGwJNlyYx7nDeQPIRykA4bU48IlBfirryyJVHwzEpbRI7pZZDNutRCx0NkNzzBJK9PlbnrmF9RAQzUrtuRTxN+6MNgaZLkxiTEXyffATJZABOexRVFgv+Gned9oDyaFw7PTkH1gmqLBZio45yVVabg6HY+Eqq8Ilw0oCzju75Ncc02hBoujSJIXkDyXEtL1ofitNuuGmqPCV12kvdpvJodPuohI5IGkG0ea6Y5KHtcszWMMSRhBIhxq8Y3fuUo35+zbGLNgSaLk1oWcojdQ1FmW6ayuriOu0lFYY2T1x02hGOri42i40TTJdMbDusO7SWobGGoZxGFDbr0Q1f1RzbaEOg6dLYrXZizK/xkS5+Ok1D4KkqrdNe5ikAIO4IDUw4JqVNAo6evEQoo3ucgCjFrNgBR/3cmmMbbQg0XZ6k6J5YxXLE4mnBcpU1ZXXaSz1FAMQfgWxFY5yccTI2i40+sX3a7ZgtpU/GVD7IPsg56bOO+rk1xzZtqkeg0RwLJESnUumvPeLiKkFDUF3PEFSXAG0TnKvPwPiBLP/R8qOmM1SHXuPpO/MOGHPJ0T+35phGGwJNl2dE0gji2xDi6TRnEp56UUMl5gyhvesGdIoRALBYYcavO+fcmmMabQg0XZ7bJt/Wpv2dkfEAVJp1AgKUeiuxWw/XNdZojlf0GoGm2+M0XT8eb93M4jKfh3ixHzWVUI2ms9CGQNPtCUhXe+pJTJT6a9pFeVSjOdbRhkDT7Qm4furMCPx+SpSP2HZQHtVojnXaZAhEJFFEPheRTPM1bPyeiHwiIiUi8mG99v4iskJEdorImyKiH780R53DhqD6cGNVCaVWC3FHWRxOo+kM2jojuA34Qik1GPjC/ByOB4GrwrQ/ADyslBoEFAM/beN4NJpWY7fYsSrw+KoON5rKo/HtGDqq0RyrtNUQnA+8aL5/EbggXCel1BdAnZAMMVbgTgXmN7e/RtORiAhOBI+/5nCjp4SydlQe1WiOZdoaPtpDKWUWQuUQ0KMV+yYBJUopr/k5G0hvrLOIXAdcB9CnT8OszNraWrKzs6mqqmqwTdN9iYyMJCMjA7u9aW2dKLHUMQRV7kNUWSzEtWNWsUZzrNKsIRCRRUA41a0/hn5QSikRUe01sPoopZ4GngaYOHFig/NkZ2cTExNDv379dLifBgClFIWFhWRnZ9O/f9PKpE6sePze4OeA8mhsVPsoj2o0xzLNGgKl1OmNbRORXBHpqZQ6KCI9gbxWnLsQiBcRmzkryAByWrF/HaqqqrQR0NRBREhKSiI/P7/Zvk6LDY86HDVUWpELQHw7Cs5pNMcqbV0jWAjMM9/PA95v6Y5KKQV8BVx8JPuHQxsBTX1a+p1wih2P8gU/l3aA8qhGc6zSVkNwP3CGiGQCp5ufEZGJIvJMoJOILAHeBk4TkWwRmWNu+j3wfyKyE2PN4Nk2jkejOSKcVgceDnscSz2FAMQ5EztrSBrNUaNNhkApVaiUOk0pNVgpdbpSqshsX62U+llIv5OUUilKKadSKkMp9anZvlspNVkpNUgpdYlSqrqxc3UFrFYr48aNC/7s3buXxYsXExcXF2w7/fTTufHGGxk3bhwjRozA6XQGt82fP7/O8d577z22bNnSbuN7/PHHGTRoECJCQUFBsF0pxc0338ygQYMYM2YMa9euBWD79u1MmDCBMWPGsGzZMgC8Xi+nn346lZWVYc/RVXFaHFSiQBnGoLTaqE0Q59Dho5rjHy061444nU7WrVtXp23v3r2cdNJJfPjhhw367927l7lz5zbYJ8B7773H3LlzGTFiRLuMb/r06cydO5eZM2fWaf/444/JzMwkMzOTFStW8Itf/IIVK1bw1FNP8eijj9KvXz9uueUWFixYwL///W+uvPJKoqKOr4xbpy3CKGDvrQK7k1JTebQ9Jag1mmOV49IQ3PnBZrYcKGu+YysY0SuWv5w7sl2P2RRLly5l4cKFfP3119xzzz0sWLCA8vJyrr/+eiorKxk4cCDPPfccCQkJzJw5k7Fjx/L111/j9Xp57rnnmDx5coNjjh8/Puy53n//fX784x8jIkydOpWSkhIOHjyI3W6nsrKSyspK7HY7JSUlfPDBB3zyyScdfflHHafVicciUFNpGAJvhVYe1XQbtNZQO+LxeIJungsvvDDYvmTJkmD7vffe26JjnXjiiZx33nk8+OCDrFu3joEDB/LjH/+YBx54gA0bNjB69GjuvPPOYP/KykrWrVvHk08+yTXXXNOqcefk5NC7d+/g54yMDHJycrjxxhu57777mDdvHn/4wx+4++67+cMf/oDFcvx9bZx2pzEjqK0AoNRXRZxWHtV0E47LGcHRfHIPJZxrCGjUNdQaSktLKSkp4ZRTTgFg3rx5XHLJ4UpTl19+OQAnn3wyZWVllJSUEB8f36Zz9unTh8WLFwOwc+dOsrOzGT58OFdddRU1NTXcfffdDBkypE3nOFZw2qLwWCz4q91YCCiPHl/uL42mMY6/R7tuSv0nVxFhzpw5jBs3jp/97GeN7GWQnp7O/v37g5+zs7NJT6+b5P3HP/6Re+65h8cee4yf/exn/P3vf68zI+nqOO2GuFxVdQn4fZQqH3FaeVTTTdCG4BgmJiaG8nJDoikuLo6EhASWLFkCwMsvvxycHQC8+eabAHz77bfExcURFxfHp59+yrp163jmmWcaHjyE8847j5deegmlFMuXLycuLo6ePQ/Hz3/99df06tWLwYMHU1lZicViwWKxHFeRQ06HCwCPpwSqSim1WIizuzp3UBrNUeK4dA0dL1x22WVce+21PPbYY8yfP58XX3wxuFg8YMAAnn/++WDfyMhIxo8fT21tLc8991zY4z322GP8/e9/59ChQ4wZM4azzz6bZ555hrPPPpuPPvqIQYMGERUVVee4SinuueeeoKG57rrruOKKK/B6vfz73//u2F/AUSTKEShgXwKeYkqsFkY4Yjp3UBrNUUKU6jB5oA5j4sSJavXq1XXatm7dyvDhwztpRJ3LzJkzeeihh5g4cWJnD+WYpCXfjU/XP8et6x7mnRE3MLjnJCZ9Po9L02dx6+zHj9IoNZqOR0TWKKUa3Ci0a0ijAZwR5oygppwqdy5VFgvxzuROHpVGc3TQrqHjgEBkj+bIcUbGA+CpLqes4hCglUc13Qc9I9BogKgIo8qqp9ZNidswBHGucOrrGs3xh54RaDQcnhFU1rgp9RvrZlp5VNNd0IZAowGcdiNnwFNbSZlZxD5eK49qugnaEGg0HNYU8vg84DMqlWnlUU13Qa8RtCPZ2dmcf/75DB48mIEDB3LLLbdQU1MTtu+BAwe4+OKLw24L5eyzz6akpOSIxvPXv/6Vhx56KGx7eno648aNY9SoUSxcuPCIjt/e3HfffZ12bqfdNATeKkprzCQ+rTyq6SZoQ9BOKKW46KKLuOCCC8jMzGTHjh243W7++Mc/Nujr9Xrp1atXg/oD4fjoo4/arBkUjl//+tesW7eOt99+m2uuuQa/39+i/Xw+X/OdjpAjMQTtNR6HxYFFgcdXbSiPIlp5VNNtOD5dQx/fBoc2tu8x00bDWfc3uvnLL78kMjKSn/zkJ4BRpObhhx+mf//+3Hnnnbz11lu88847uN1ufD4fL774InPnzmXTpk1UVlZy9dVXs2nTJoYOHcqBAwd44oknmDhxIv369WP16tW43W7OOussZsyYwdKlS0lPT+f999/H6XTy3//+l6effpqamhoGDRrEyy+/3OJ6AcOHD8dms1FQUMB1113H/v37qaqq4pZbbuG6664DwOVy8fOf/5xFixbxxBNP8OWXX/LBBx/g8Xg48cQTeeqppxARZs6cyfjx41myZAkVFRW89NJL/O1vf2Pjxo1ceuml3HPPPQC88sorPPbYY9TU1DBlyhSefPJJ/vjHPwbVW0eOHMmrr74atp/Vam0wng8//JCFCxdis9mYPXt22FlQc4gITgSPrxqPt4o4m00rj2q6DXpG0E5s3ryZCRMm1GmLjY2lT58+7Ny5E4C1a9cyf/58vv766zr9nnzySRISEtiyZQt33303a9asCXuOzMxMbrzxRjZv3kx8fDwLFiwA4KKLLmLVqlWsX7+e4cOH8+yzLa/4uWLFCiwWCykpKTz33HOsWbOG1atX89hjj1FYaJRrrKioYMqUKaxfv54ZM2Zw0003sWrVKjZt2oTH46mjrOpwOFi9ejXXX389559/Pk888QSbNm3ihRdeoLCwkK1bt/Lmm2/y3XffsW7dOqxWK6+++ir3339/UL311VdfbbRf/fEMHz6cd999l82bN7NhwwbuuOOOFl97fZxiweOvMZVHHUd8HI2mq3F8zgiaeHLvTM444wwSExtGonz77bfccsstAIwaNYoxY8aE3b9///6MGzcOgAkTJrB3714ANm3axB133EFJSQlut5s5c+aE3T+Uhx9+mFdeeYWYmBjefPNNRITHHnuMd999F4D9+/eTmZlJUlISVquVH/zgB8F9v/rqK/7+979TWVlJUVERI0eO5NxzzwUMATuA0aNHM3LkyKB43YABA9i/fz/ffvsta9asYdKkSYBRwyE1NbXB+L744otG+4WOJy4ujsjISH76058yd+5c5s6d2+y1N0YUVir9tZSilUc13Yvj0xB0AiNGjGjg8y8rKyMrK4tBgwaxdu1aoqOj23SOiIiI4Hur1YrH4wHg6quv5r333mPs2LG88MILLco0/vWvf82tt94a/Lx48WIWLVrEsmXLiIqKYubMmVRVVQGGoJ3VagWgqqqKG264gdWrV9O7d2/++te/BvuFjtFisdQZr8Viwev1opRi3rx5/O1vf2tyfE31Cx2PzWZj5cqVfPHFF8yfP5/HH3+cL7/8stnrD4fTYsXjraDUYiFdK49quhHaNdROnHbaaVRWVvLSSy8BxiLmb37zG66++upm/fXTp0/nrbfeAmDLli1s3Ni69Y3y8nJ69uxJbW1t0H3SWkpLS0lISCAqKopt27axfPnysP0CN/3k5GTcbneLFrxDOe2005g/fz55eXkAFBUVsW/fPgDsdju1tbXN9gvF7XZTWlrK2WefzcMPP8z69etbNZ5QnGLHo3yUWi3EaeVRTTeiTYZARBJF5HMRyTRfExrp94mIlIjIh/XaXxCRPSKyzvwZ15bxdCYiwrvvvsvbb7/N4MGDGTJkCJGRkS2KhLnhhhvIz89nxIgR3HHHHYwcOZK4uJaHLt59991MmTKF6dOnM2zYsCMa/5lnnonX62X48OHcdtttTJ06NWy/+Ph4rr32WkaNGsWcOXOCrpuWMmLECO655x5mz57NmDFjOOOMMzh48CBgSFyPGTOGK664osl+oZSXlzN37lzGjBnDjBkz+Oc//9n6izdxWhx4RIxaBBFhv8oazXFJm2SoReTvQJFS6n4RuQ1IUEr9Pky/04Ao4OdKqbkh7S8AHyqlWvVYebzJUPt8Pmpra4mMjGTXrl2cfvrpbN++HYdDL1i2By39btzy5hx2lu8jy27n5gEXce1Jx08FNo0GGpehbusawfnATPP9i8BioIEhUEp9ISIz67drDCorK5k1axa1tbUopXjyySe1EegEnNYIcs21h7iohgvYGs3xSlsNQQ+lVGC+fgjocQTHuFdE/gx8AdymlKoO10lErgOuA6Oo+vFETEwM9Wc4mqOP0+ak2mJ4S7XyqKY70ewagYgsEpFNYX7OD+2nDB9Ta/1MtwPDgElAImFmEyHHf1opNVEpNTElRevEa9qf0ExirTyq6U40OyNQSp3e2DYRyRWRnkqpgyLSE8hrzclDZhPVIvI8cGtT/TWajsQZkjsQF6kXizXdh7aGjy4E5pnv5wHvt2Zn03ggRi7/BcCmNo5HozlinI7DeR7xEfGdNxCN5ijTVkNwP3CGiGQCp5ufEZGJIvJMoJOILAHeBk4TkWwRCaS+vioiG4GNQDJwTxvHo9EcMU77YUOglUc13Yk2GQKlVKFS6jSl1GCl1OlKqSKzfbVS6mch/U5SSqUopZxKqQyl1Kdm+6lKqdFKqVFKqSuVUu62XU7ncu+99zJy5EjGjBnDuHHjWLFiRZuP+fnnnzNhwgRGjx7NhAkTGs2afeSRR6isrGzz+QJcc801pKamMmrUqDrtRUVFnHHGGQwePJgzzjiD4uJiABYsWMDIkSM56aSTghpFu3bt4tJLL223MXU0UWYSmQ208qimW6Ezi9uJZcuW8eGHH7J27Vo2bNjAokWL6N27d5uPm5yczAcffMDGjRt58cUXueqqq8L2a29DcPXVV/PJJ580aL///vs57bTTyMzM5LTTTuP++w1dp3/961+sWrWKn//857z22msA3HHHHUHF0a6AMyIWgDixa+VRTbfiuNQaemDlA2wr2tauxxyWOIzfT240qImDBw+SnJwc1NdJTk4GDHnqxx57jPfeew8wnvCffPJJ3n33XVwuF7/4xS/46KOP6NmzJ/fddx+/+93vyMrK4pFHHuG8885j/PjxwXOMHDkSj8dDdXV1HR2fxx57jAMHDjBr1iySk5P56quveP3117nvvvtQSnHOOefwwAMPAIak9LXXXstnn31GWloab7zxBuGisE4++eSgqF0o77//flDLaN68ecycOZMHHngAi8VCdXU1lZWV2O12lixZQlpaGoMHD27V77kziTLdQfGWiGZ6ajTHF3pG0E7Mnj2b/fv3M2TIEG644Yag1PSsWbPYtm0b+fn5ADz//PNcc801gCGnfOqpp7J582ZiYmK44447+Pzzz3n33Xf585//3OAcCxYs4IQTTqhjBABuvvlmevXqxVdffcVXX33FgQMH+P3vf8+XX37JunXrWLVqVdAQVVRUMHHiRDZv3swpp5zCnXe2Lns2Nzc3qCialpZGbm4uALfffjunn346H3zwAZdffjl33303f/rTn1p17M7GaRqCOO0W0nQzjssZQVNP7h2Fy+VizZo1LFmyhK+++opLL72U+++/n6uvvpqrrrqKV155hZ/85CcsW7YsKEzncDg488wzAUO2OSIiArvdzujRoxs8jW/evJnf//73fPbZZ82OZdWqVcycOTP4pH/FFVfwzTffcMEFF2CxWIJ++yuvvJKLLrroiK9ZRIIulDPOOIMzzjgDgJdeeomzzz6bHTt28NBDD5GQkMCjjz7a4mI5nYUz2sgmjo3SeSqa7oWeEbQjVquVmTNncuedd/L4448HC8f85Cc/4ZVXXuH111/nkksuwWYz7K/dftgXHSrbHJBsDpCdnc2FF17ISy+9xMCBA9t1zCLC/v37GTduHOPGjeM///lPk/179OgRFH87ePBgg1oClZWVvPDCC9x444385S9/4cUXX2TGjBlHrIp6NAmuESR2HXeWRtMeaEPQTmzfvp3MzMzg53Xr1tG3b18AevXqRa9evbjnnnuCpSxbSklJCeeccw73338/06dPb7RfTEwM5eVG0fXJkyfz9ddfU1BQgM/n4/XXX+eUU04BwO/3B6WjX3vtNWbMmEHv3r1Zt24d69at4/rrr29yPOeddx4vvvgiAC+++CLnn18nwZwHH3yQm2++GbvdjsfjQUSwWCztupDdUQQihXQOgaa7oQ1BO+F2u5k3bx4jRoxgzJgxbNmyhb/+9a/B7VdccQW9e/dutULq448/zs6dO7nrrruCT+0Bjf5QrrvuOs4880xmzZpFz549uf/++5k1axZjx45lwoQJwRt2dHQ0K1euZNSoUXz55Zdh1yIALr/8cqZNm8b27dvJyMgIlr+87bbb+Pzzzxk8eDCLFi3itttuC+5z4MABVq5cyQUXXADAL3/5SyZNmsR//vMffvSjH7XqujuDgCGINWcGGk13oU0y1J1FV5Shvummmxg/fjw//elPO3UcLpcLt7tLp2u0mpZ+N5RSPL3hac7ufza9Y9se+qvRHGt0lAy1pgVMmDCB6Oho/vGPf3T2UDRNICL8fOzPO3sYGs1RRxuCo8CaNWs6ewhButtsQKPRNM9xtUbQFd1cmo5Ffyc0muY5bgxBZGQkhYWF+h9fE0QpRWFhIZGRkZ09FI3mmOa4cQ1lZGSQnZ0dzODVaMB4QMjIyOjsYWg0xzTHjSGw2+3079+/s4eh0Wg0XY7jxjWk0Wg0miNDGwKNRqPp5mhDoNFoNN2cLplZLCL5wL4j3D0ZKGjH4XQV9HV3L7rrdUP3vfaWXHdfpVQDed0uaQjagoisDpdifbyjr7t70V2vG7rvtbflurVrSKPRaLo52hBoNBpNN6c7GoKnO3sAnYS+7u5Fd71u6L7XfsTX3e3WCDQajUZTl+44I9BoNBpNCNoQaDQaTTenWxkCETlTRLaLyE4Rua35PbomIvKciOSJyKaQtkQR+VxEMs3XhM4cY0cgIr1F5CsR2SIim0XkFrP9uL52EYkUkZUist687jvN9v4issL8vr8pIo7OHmtHICJWEfleRD40Px/31y0ie0Vko4isE5HVZtsRf8+7jSEQESvwBHAWMAK4XERGdO6oOowXgDPrtd0GfKGUGgx8YX4+3vACv1FKjQCmAjeaf+Pj/dqrgVOVUmOBccCZIjIVeAB4WCk1CCgGOrdOasdxC7A15HN3ue5ZSqlxIbkDR/w97zaGAJgM7FRK7VZK1QBvAOd38pg6BKXUN0BRvebzgRfN9y8CFxzNMR0NlFIHlVJrzfflGDeHdI7za1cGgdJzdvNHAacC88324+66AUQkAzgHeMb8LHSD626EI/6edydDkA7sD/mcbbZ1F3oopQ6a7w8BPTpzMB2NiPQDxgMr6AbXbrpH1gF5wOfALqBEKeU1uxyv3/dHgN8BfvNzEt3juhXwmYisEZHrzLYj/p4fN/UINC1HKaVE5LiNGxYRF7AA+JVSqsx4SDQ4Xq9dKeUDxolIPPAuMKxzR9TxiMhcIE8ptUZEZnbycI42M5RSOSKSCnwuIttCN7b2e96dZgQ5QO+QzxlmW3chV0R6ApiveZ08ng5BROwYRuBVpdQ7ZnO3uHYApVQJ8BUwDYgXkcDD3vH4fZ8OnCciezFcvacCj3L8XzdKqRzzNQ/D8E+mDd/z7mQIVgGDzYgCB3AZsLCTx3Q0WQjMM9/PA97vxLF0CKZ/+Flgq1LqnyGbjutrF5EUcyaAiDiBMzDWR74CLja7HXfXrZS6XSmVoZTqh/H//KVS6gqO8+sWkWgRiQm8B2YDm2jD97xbZRaLyNkYPkUr8JxS6t7OHVHHICKvAzMxZGlzgb8A7wFvAX0wJLx/qJSqv6DcpRGRGcASYCOHfcZ/wFgnOG6vXUTGYCwOWjEe7t5SSt0lIgMwnpQTge+BK5VS1Z030o7DdA3dqpSae7xft3l975ofbcBrSql7RSSJI/yedytDoNFoNJqGdCfXkEaj0WjCoA2BRqPRdHO0IdBoNJpujjYEGo1G083RhkCj0WiOUUTkElNI0C8iLapHLCI+U4xunYi0KEReGwLNMU3Il3qTiLwtIlHHwJhmisiJbTxGLxGZ33zPOvtcLSKPm++vF5Eft2UMmmML83v1Qr3mTcBFwDetOJTHFKMbp5Q6ryU7aEOgOdYJfKlHATXA9S3ZKSSztCOYCbTKENQfj1LqgFLq4sb6N4dS6j9KqZeOdH9N10AptVUptb1+u6kt9aCIrBKRDSLy87acRxsCTVdiCTBIRM419ea/F5FFItIDQET+KiIvi8h3wMsi0k9ElojIWvPnRLPfTBH5WkTeF5HdInK/iFwhhqb/RhEZaPZLEZEF5j/bKhGZborZXQ/82pypnBSuX7jxhF6IObZN5vurReQdEflEDC35v4f0+4mI7BCRlRiSCoQc+1bz/SDz97DevM7A+H8bcqMI1CiIFpH/mX03icilHfKX0nQ0PwVKlVKTgEnAtSLS39wWKSKrRWS5iFzQkoNp0TlNl8B8oj4L+AT4FphqCmv9DEN98jdm1xEYglwe0410hlKqSkQGA68DAT/rWGA4hlz3buAZpdRkMYrZ/BL4FYZuzcNKqW9FpA/wqVJquIj8B3ArpR4yx/Za/X7mseuMp5lLHIehlloNbBeRf2HUV7gTmACUYkgnfB9m31eB+5VS74pIJGARkdnAYAwNGgEWisjJQApwQCl1jjn2uGbGpelgRGQFEAG4gEQxVGQBfq+U+rSR3WYDY0QkMKuMw/h77wH6moJ0A4AvRWSjUmpXU2PQhkBzrOMM+cdYgqElNBR4UwxhLQfGlz/AwpCbrh14XETGAT5gSEi/VQHJXhHZBXxmtm8EZpnvTwdGyGH10lgxlE3r01S/hS0wAmAUFCk1x7MF6IshEbJYKZVvtr9Z7xoQQ3MmXSn1LoBSqspsn41xswgYDhfGjWIJ8A8ReQD4UCm1pAVj03QgSqkpEJTJuFopdXULdhPgl+EMRYgg3W4RWYzxgKENgaZL41FKjQttMJ+W/6mUWmj+8/w1ZHNFyPtfY2gtjcVwg1aFbAvVnvGHfPZz+P/CgjHzCN2PkBs+LehXUb9zI4SOx0fb/zcF+JtS6qkGG0ROAM4G7hGRL5RSd7XxXJqjz6fAL0TkS6VUrYgMwVBZdQCVSqlqEUnGcCf+vakDgV4j0HRN4jgsLTyvmX4HlVJ+4CoMUbbW8BmGmwgAc2YBUA7EtKBfW1kBnCIiSWLIa19Sv4NZiS074AsWkQjTJfYpcE1gZiIi6SKSKiK9MG4UrwAPAie001g1HYCIXCgi2Riy4v8TkcAM4BlgC7DWXGt6CuPhYTiwWkTWY7gS71dKbWnuPHpGoOmK/BV4W0SKgS+B/o30exJYIEaY5Se0/Ok8wM3AEyKyAeN/5RuMheIPgPkicj6GAWisX5tQSh0Ukb8Cy4ASYF0jXa8CnhKRu4Ba4BKl1GciMhxYZs5M3MCVwCDgQRHxm31/0dZxatoHpdRiYHG9tnc5rDQa2u7HUNb9Q71NS4HRrT23Vh/VaDSabo52DWk0Gk03RxsCjUaj6eZoQ6DRaDTdHG0INBqNppujDYFGo9F0c7Qh0Gg0mm6ONgQajUbTzfl/yabYiB7EGakAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(np.arange(100000,100050,1), reverse_top10fft[100000:100050], label = \"FFT top-10%\")\n",
+ "plt.plot(np.arange(100000,100050,1), concr[100000:100050], label = \"Original Parameters\")\n",
+ "plt.plot(np.arange(100000,100050,1), reverse_top10wv[100000:100050], label = \"Sym2 top-10%\")\n",
+ "\n",
+ "plt.title('Parameter Values') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.legend()\n",
+ "plt.draw()\n",
+ "plt.savefig(\"ParametersWaveletHaar.png\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 203,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0000, 0.0649, 0.0881, 0.0000, 0.0464, 0.0347, -0.0472, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, -0.0365, 0.0000,\n",
+ " -0.0422, -0.0344, 0.0372, -0.0823, 0.0000, 0.0764, -0.1654, 0.0000,\n",
+ " 0.0000, -0.0363, -0.0769, 0.0896, 0.0000, 0.0955, 0.0000, -0.0843,\n",
+ " 0.0000, -0.0387, 0.1598, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
+ " 0.0000, 0.0000])"
+ ]
+ },
+ "execution_count": 203,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "top10_og[100000:100050]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 204,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACHdklEQVR4nO2ddXhUV/rHP2cmE3d3gQRKEkJwiluLLAXqQlu69a1uZbdUfvXutt1u3d2FGqVdWkpLKe4ElwQIRIm7Z87vjzsTJsnESCAy5/M8eSZz77l3zk1m5r3nle8rpJQoFAqFwnbRdfcEFAqFQtG9KEOgUCgUNo4yBAqFQmHjKEOgUCgUNo4yBAqFQmHjKEOgUCgUNo4yBApFL0cIMVkIkd7d81D0XpQhUHQrQohUIUSlEKJMCHFCCPGhEMK1u+fVGkKISCGEFELYddH5HIUQRUKIqVb2vSCE+KYrXkehaAllCBQ9gfOklK7AMGAE8FBHDhYavea93NSASCmrgK+Aq5uM0wOXAx+dudkpbJFe8+FR9H2klBnAz0C8EMJLCPGTECJXCFFo+j3UPFYIsUoI8ZQQYh1QAfQTQvxVCLFfCFEqhDgihLjJYvxkIUS6EOKfQogcIUSWEGK+EGK2EOKQEKJACPGAxXidEGKREOKwECJfCLFYCOFt2r3a9FhkWsmcbTrmWtPrFwohlgshIizOJ4UQtwohkoFkK5f/EXChEMLZYtsMtM/oz61dW1NMrxVt8fxDIcSTFs/nCCGSTKuQ9UKIBIt99wkhMkyvc1AIMa2l11H0HZQhUPQYhBBhwGxgB9p78wMgAggHKoFXmxxyFXAj4AYcA3KAOYA78FfgBSHEMIvxgYAjEAI8DLwDXAkMByYA/yeEiDKNvR2YD0wCgoFC4DXTvommR08ppauUcoMQYh7wAHAB4AesAb5oMt/5wGggtum1SynXA1mm4y2v73MpZV07rq1dCCGGAu8DNwE+wFvAUiGEgxBiIHAbMFJK6YZmiFI7+hqKXoiUUv2on277QfuiKQOK0L7MXwecrIxLBAotnq8CHm/j3EuAO02/T0YzJnrTczdAAqMtxm8D5pt+3w9Ms9gXBNQCdkCk6Vg7i/0/A9dZPNehrVQiTM8lMLWN+T4E/Gr63d10/NB2Xlu6xT4JRFs8/xB40vT7G8ATTc51EM3gRaMZnOmAobvfG+rnzP2oFYGiJzBfSukppYyQUt4ipawUQjgLId4SQhwTQpSguWM8TX5zM2mWJxFCzBJCbDS5eYrQVhe+FkPypZT1pt8rTY8nLPZXAuZAdQTwvcl9UoRmGOqBgBauIQJ4yWJ8ASDQVh9W52uFT4ApQohg4CLgsJRyRzuvrb1EAPeY52k6VxgQLKVMAf4OPArkCCG+NM1F0cdRhkDRU7kHGIh2x+7OSXeMsBjTIJ0rhHAAvgWeAwKklJ7AsibjO0IaMMtkoMw/jlKLY1iT7E0Dbmoy3klqLp9m87WGlPIYmkvpSjS30EeneG0VgGWsIbDJPJ9qMk9nKeUXpjl8LqUcj2YwJPBMa3NW9A2UIVD0VNzQ7tCLTEHaR9oYbw84ALlAnRBiFnBuJ17/TeApc8BXCOFnigNgeg0j0K/J+PuFEHGm8R5CiItP4XU/QvPTjwM+M23r6LUlAVcIIfRCiJlobh8z7wA3CyFGm7KtXIQQfxFCuAkhBgohppoMTxXa3994Cteg6GUoQ6DoqbwIOAF5wEbgl9YGSylLgTuAxWiB3SuApZ14/ZdMx/8qhCg1zWG06bUqgKeAdSb3yhgp5fdod89fmlxZe4BZp/C63wLewO9SyqxTvLY7gfPQ4i4L0OIJmM61FbgBLfBeCKQA15h2OwBPo/3NswF/4P5TuAZFL0NIqRrTKBQKhS2jVgQKhUJh4yhDoFAoFDaOMgQKhUJh4yhDoFAoFDZOl6gnnml8fX1lZGRkd09DoVAoehXbtm3Lk1L6Nd3eKw1BZGQkW7du7e5pKBQKRa9CCHHM2nblGlIoFAobRxkChUKhsHGUIVAoFAobp1fGCKxRW1tLeno6VVVV3T0VRQ/C0dGR0NBQDAZDd09Foeix9BlDkJ6ejpubG5GRkQhxqoKTir6ElJL8/HzS09OJiopq+wCFwkbpM66hqqoqfHx8lBFQNCCEwMfHR60SFYo26DOGAFBGQNEM9Z5QKNqmTxkChULRxRxZBTn7u3sWitOMMgRdiF6vJzExseEnNTWVVatW4eHh0bBt+vTp3HrrrSQmJhIbG4uTk1PDvm+++abR+ZYsWcK+ffu6bH6vvvoq0dHRCCHIy8tr2C6l5I477iA6OpqEhAS2b98OwMGDBxk+fDgJCQls2LABgLq6OqZPn05FRUWXzUvRQzHWw+KrYeWT3T0TxWmmzwSLewJOTk4kJSU12paamsqECRP46aefmo1PTU1lzpw5zY4xs2TJEubMmUNsbGyXzG/cuHHMmTOHyZMnN9r+888/k5ycTHJyMps2beJvf/sbmzZt4q233uKll14iMjKSO++8k2+//ZY33niDK6+8EmdnZ+svoug7ZO+CqmIoslqMquhDqBVBD2X9+vUsXbqUf/zjHyQmJnL48GGSkpIYM2YMCQkJnH/++RQWFgIwefJk7rzzThITE4mPj2fz5s1Wzzl06FCsaTT98MMPXH311QghGDNmDEVFRWRlZWEwGKioqKCiogKDwUBRURE//vgjV1999em8dEVP4ega7bHoePfOQ3Ha6ZMrgsd+3Mu+zJIuPWdssDuPnBfX6pjKykoSExMBiIqK4vvvvwdgzZo1DdsvvvhiHnzwwTZfb+zYscydO5c5c+Zw0UUXAZCQkMArr7zCpEmTePjhh3nsscd48cUXAaioqCApKYnVq1dz7bXXsmfPnnZfW0ZGBmFhYQ3PQ0NDycjI4NZbb+Xqq6+murqat956iyeeeIIHHngAnU7dP9gER1drj1XFUFkETp7dORvFaaRPGoLuwpprCGjRNdQRiouLKSoqYtIkrQ/5woULufjik73RL7/8cgAmTpxISUkJRUVFeHp6duo1w8PDWbVqFQApKSmkp6czaNAgrrrqKmpqanjiiScYMGBAp15D0UOpr4XjG8A1EMqytVWBMgRdyqs7XmWQzyCmhU/r7qn0TUPQ1p17X6RpmqQQghkzZnDixAlGjBjBu+++2+KxISEhpKWlNTxPT08nJCSk0ZgHH3yQJ598kpdffpnrr7+eyMhIHnjgAT777LOuvRBFzyAzCWrKYOT1sO5FzRAEJXT3rPoUn+3/jJGBI3uEIVBr/B6Mm5sbpaWlAHh4eODl5cWaNZrf9pNPPmlYHQB89dVXAKxduxYPDw88PDxYvnw5SUlJrRoBgLlz5/Lxxx8jpWTjxo14eHgQFBTUsP/PP/8kODiYmJgYKioq0Ol06HQ6lTnUlzn6p/aYuEB7VHGCLqW6vpqy2jLSStPaHnwG6JMrgr7CZZddxg033MDLL7/MN998w0cffcTNN99MRUUF/fr144MPPmgY6+joyNChQ6mtreX999+3er6XX36ZZ599luzsbBISEpg9ezbvvvsus2fPZtmyZURHR+Ps7NzovFJKnnzyyQZDc+ONN7JgwQLq6up44403Tu8fQNF9pK4B/zjwjQF7N2UIupjCKi3RI700HaM0ohPdfE8upez0DzATOAikAIus7HcAvjLt3wRENtkfDpQB97bn9YYPHy6bsm/fvmbbbIVJkybJLVu2dPc0eiy2/N44JWqrpHwiQMpl/9Sev3a2lJ9f3r1z6mPsydsj4z+Ml/Efxsvssuwz9rrAVmnlO7XTZkgIoQdeA2YBscDlQoimie/XAYVSymjgBeCZJvufB37u7FwUCkUXkL4V6iohcoL23DNcrQi6mILKgobfj5d2/9+2K9Yjo4AUKeURKWUN8CUwr8mYecBHpt+/AaYJU3RTCDEfOArs7YK52CSrVq1ixIgR3T0NRV8hdQ0gIHKc9twzXCsq01bvii6goOqkIUgvTe/GmWh0hSEIASwjHummbVbHSCnrgGLARwjhCtwHPNbWiwghbhRCbBVCbM3Nze2CaSsUCqscXaNlCDl5ac89w6G6BKqKunVafQmzIRCIPrMi6AyPAi9IKcvaGiilfFtKOUJKOcLPz+/0z0yhsEVqKyF9c4Nb6N3d7/KHLNf2KfdQl1FYVYiD3oFQt9AekTnUFVlDGUCYxfNQ0zZrY9KFEHaAB5APjAYuEkI8C3gCRiFElZTy1S6Yl0Kh6Chpm6C+BqImsjZjLS9tf4mzveOZAqZagiHdPcM+QX5VPt6O3oS7hXO8pPsNbFcYgi1AjBAiCu0L/zLgiiZjlgILgQ3ARcBKUwR7gnmAEOJRoEwZAYWiGzm6BoSeyuChPLlc05Q6WpWj7VMrgi6joKoAb0dvwtzC2JW7Cyllt/bO6LRryOTzvw1YDuwHFksp9wohHhdCzDUNew8tJpAC3A0s6uzr9kTS09OZN28eMTEx9O/fnzvvvJOamhqrYzMzMxs0hFpj9uzZFBUVndJ8Hn30UZ577jmr20NCQhpE6pYuXXpK5+9q/vWvf3X3FBRHV0PwUN48+BkZZRmMCx5HdkUOFY7uUKhUSLsKsyEIdw+ntLaUouqibp1Pl8QIpJTLpJQDpJT9pZRPmbY9LKVcavq9Skp5sZQyWko5Skp5xMo5HpVSNv/W6iVIKbnggguYP38+ycnJHDp0iLKyMqsCc3V1dQQHBzfrP2CNZcuWdVozyBp33XUXSUlJfP3111x77bUYjcZ2HVdfX9/lczFzKobgdM7H5qgug8ztHApN4OO9H3N+9PlcOOBCAFI9Q9SKoAuxXBEA3R4n6O5gcZ9h5cqVODo68te//hXQmtS88MILvP/++1RUVPDhhx8yd+5cpk6dyrRp00hNTSU+Ph7QlEMvueQSYmNjOf/88xk9ejRbt24FIDIykry8PFJTUxk0aBA33HADcXFxnHvuuVRWVgLwzjvvMHLkSIYMGcKFF17YIemHQYMGYWdnR15eHvPnz2f48OHExcXx9ttvN4xxdXXlnnvuYciQIWzYsIHHH3+ckSNHEh8fz4033mguCmTy5MncddddjBgxgkGDBrFlyxYuuOACYmJieOihhxrO9+mnnzJq1CgSExO56aabqK+vZ9GiRQ3qrQsWLGhxnLX5LFq0iNjYWBISErj33ntP9V+oOL4Ro7GOxypTcLN34+7hdxPlHgXAUVcvZQi6CCklBZUFeDtpMQLo/lqCvikx8fMiyN7dtecMHAyznm5x9969exk+fHijbe7u7oSHh5OSkgLA9u3b2bVrF97e3qSmpjaMe/311/Hy8mLfvn3s2bOnQbK6KcnJyXzxxRe88847XHLJJXz77bdceeWVXHDBBdxwww0APPTQQ7z33nvcfvvt7bqsTZs2odPp8PPz4/3338fb25vKykpGjhzJhRdeiI+PD+Xl5YwePZr//ve/AMTGxvLwww8DcNVVV/HTTz9x3nnnAWBvb8/WrVt56aWXmDdvHtu2bcPb25v+/ftz1113kZOTw1dffcW6deswGAzccsstfPbZZzz99NO8+uqrDeqt+/fvtzru6quvbjSf/Px8rrvuOg4cOIAQ4pTdaAogdTXfuHuwqzSVf43/F56OnjgbnNEJHUcdHKFot1ZLoPpAd4ry2nJqjDX4OPoQ4haCQDReEeQcgNpyCBne8km6mL5pCHoo55xzDt7e3s22r127ljvvvBOA+Ph4EhKsqzxGRUU1GInhw4c3GJM9e/bw0EMPUVRURFlZGTNmzGhzLi+88AKffvopbm5ufPXVVwghePnllxt6KKSlpZGcnIyPjw96vZ4LL7yw4dg//viDZ599loqKCgoKCoiLi2swBHPnamGhwYMHExcX1yBe169fP9LS0li7di3btm1j5MiRgNbDwd/fv9n8fv/99xbHWc7Hw8MDR0dHrrvuOubMmcOcOXPavHaFdXKPruJFb09GB45iTj/t72ivtyfUNZSj0gg1pVBZCM7N38OK9mOuIfB29MZB70CASwBpJSZDkLYFPjkfPELh1o1nbE590xC0cud+uoiNjW3m8y8pKeH48eNER0ezfft2XFxcOvUaDg4ODb/r9foG19A111zDkiVLGDJkCB9++GFDD4HWuOuuuxq5UVatWsVvv/3Ghg0bcHZ2ZvLkyVRVVQGaoJ1erwegqqqKW265ha1btxIWFsajjz7aMM5yjjqdrtF8dToddXV1SClZuHAh//73v1udX2vjLOdjZ2fH5s2b+f333/nmm2949dVXWblyZZvXr2hCVTHP1GVQ7eDKQ2MeapTBEuURxdH8g9qTouPKEHQSS0MAEOYWpq0I0rfCpxdoBrck84zOScUIuohp06ZRUVHBxx9/DGhBzHvuuYdrrrmmzf6+48aNY/HixQDs27eP3bs75tYqLS0lKCiI2traU+4PUFxcjJeXF87Ozhw4cICNG63fjZi/9H19fSkrK2tXwNuSadOm8c0335CTo6UkFhQUcOyYlo1iMBiora1tc5wlZWVlFBcXM3v2bF544QV27tzZofkoNFYnvcdyF2duiJhNpEdko31RHlEcq86nHlT/4i4gvyofOGkIwt3COV50RFsJOPvAiOugulgr7jtDKEPQRQgh+P777/n666+JiYlhwIABODo6tisT5pZbbiE3N5fY2Fgeeugh4uLi8PDwaPdrP/HEE4wePZpx48Zx1llnndL8Z86cSV1dHYMGDWLRokWMGTPG6jhPT09uuOEG4uPjmTFjRoPrpr3Exsby5JNPcu6555KQkMA555xDVlYWoElcJyQksGDBglbHWVJaWsqcOXNISEhg/PjxPP/88x2/eBunsq6SfyV/Sb/aOq4dc3+z/ZHukdQYa8m006uAcRfQdEUQKnUU1JZQ7uwF1/wEIcO0gaXZZ25S1iRJe/pPX5Ohrqurk5WVlVJKKVNSUmRkZKSsrq7u5ln1HXrze+NM8NqO12T8h/Fy84fTrO7flr1Nxn8YL//8b5SU/7v3DM+u7/Fm0psy/sN4WVNXI2XGdrn8hSgZ/2G83J/6hzbg0AopH3GX8tiGLn9tWpCh7psxgl5GRUUFU6ZMoba2Fiklr7/+Ovb29t09LYUNkFGWwft73mNmWTkj48+1OibKw5RC6ubDRLUi6DQFVQW42bthOLEXPp5PmIs7AGnUchaAqyl5ouzEGZuTMgQ9ADc3t4a6AYXiTPLfrf9FJ+GegiIItS5l7uXohaeDJ0d1QrmGuoCCqgJ8HH3g62vAwY2wSxfDz5ec1BxyC9QeS8+cIVAxAoXCRtmYtZEVx1ZwvWc8gfX1EJTY4tgojyiO6k2GQPUl6BQFVQV4G1yh8CiMuQVX/0F4O3qfrCVw9gWhh7IzFyNQhkChsEFqjbU8s/kZQlxDWFhh1HoOtJIWGuURRaqxEmrKoKKgxXGKtimoKsDbbEuDEwGLFFIAnU5zD6kVgUKhOJ18deArUopS+OfIf+KQvavV1QBAlHsUBfWVFOl0KoW0kxRUFeBdUw0ICNSKRxsZAgDXALUiUCgUp4/8ynxeT3qdscFjmeIzBApTIXhoq8eYA8apBjsVJ+gE9cZ6CqsK8a4oBL+B4OAKaLUE2eXZ1NSb1IrdAtWKoLfy1FNPERcXR0JCAomJiWzatKnT51yxYgXDhw9n8ODBDB8+vMWq2RdffLFDYnNtce211+Lv798gjGemoKCAc845h5iYGM455xwKCwsB+Pbbb4mLi2PChAnk52sFM4cPH+bSSy/tsjkpuoZXdrxCZV0l9426D5G9S9toclG0REPmkMGgDEEnKKouQiLxLs5qZHxD3UKRSNLLTP2L1Yqgd7JhwwZ++umnBmG53377jbCwsLYPbANfX19+/PFHdu/ezUcffcRVV11ldVxXG4JrrrmGX375pdn2p59+mmnTppGcnMy0adN4+mlNzuOVV15hy5Yt3HTTTXz++eeAJoD35JNPdtmcFJ1nb95evkv+jgWDFtDPox9kJWk72nANBbsGY9AZOOrkqgxBJzAXk3lVFjcyBOHumgppg+aQWyCU50F93RmZlzIEXURWVha+vr4N+jq+vr4EBwezcuVK5s+f3zBuxYoVnH/++YAmp/yPf/yDuLg4pk+fzubNm5k8eTL9+vVraBYzdOhQgoODAYiLi6OyspLq6upGr/3yyy+TmZnJlClTmDJlCgBffPEFgwcPJj4+nvvuu69hrKurK3fddRdxcXFMmzaN3Nxcq9czceJEqwJ5P/zwAwsXLgRg4cKFLFmyBNC0hKqrq6moqMBgMLBmzRoCAwOJiYnp6J9ScZowSiP/3vxvvB29uXnIzdrGzKQ2A8UAdjo7ItwjOOrkcmqGwGiEpM+hvrbjx/YhzIbAp76+kSFo1pfANQCQUJ5zRubVJ+sIntn8DAcKDnTpOc/yPov7Rt3X4v5zzz2Xxx9/nAEDBjB9+nQuvfRSJk2axJQpUxokJPz8/Pjggw+49tprASgvL2fq1Kn85z//4fzzz+ehhx5ixYoV7Nu3j4ULFzYoeZr59ttvGTZsWCMxN4A77riD559/nj/++ANfX18yMzO577772LZtG15eXpx77rksWbKE+fPnU15ezogRI3jhhRd4/PHHeeyxx3j11fZ3Bz1x4kSDomhgYCAnTmh+zPvvv5/p06cTHBzMp59+ysUXX8yXX37Z7vMqTj+f7/+cnbk7eWLcE7jaa75pMne0GR8wE+URRXLx8VMLFh9dBUv+pmnpDGhbHbev0iAvYQQCTrpdvRy8cDW4nuxL4BqgPZadAPfg0z4vtSLoIlxdXdm2bRtvv/02fn5+XHrppXz44YcIIbjqqqv49NNPKSoqYsOGDcyaNQvQtPtnzpwJaLLNkyZNwmAwMHjw4Eb9CkDrd3Dffffx1ltvtTmXLVu2MHnyZPz8/LCzs2PBggWsXr0a0O7czX77K6+8krVr157yNQshGlQqzznnHLZt28aPP/7IDz/8wOzZszl06BAXXXQRN9xwQ5e6rRQdJyknif9u/S+TwyYzt7/pBqOiQPtSb8MtZCbSPZI0WUvtqdQSnNinPZZbX4HaCg2GwLMf2J8UoxRCNM4cOsNFZX1yRdDanfvpRK/XM3nyZCZPnszgwYP56KOPuOaaa/jrX//Keeedh6OjIxdffDF2dtqf3WAwNHyRWso2myWbzaSnp3P++efz8ccf079//y6dsxCCtLS0hn4CN998MzfffHOL4wMCAsjKyiIoKIisrKxmvQTM3diWL1/OnDlz+O677/jmm2/47LPPGprnKM4shVWF3PvnvQS4BPDkuCfRCdP9X5ZJqbWNQLGZKI8o6pGkUUO/inxw8W3/JHL3a48V+e0/pg+SX5mPTko8goY12xfmFsbBQpPcd8OK4MwEjNWKoIs4ePAgycnJDc+TkpKIiIgAIDg4mODgYJ588smGVpbtpaioiL/85S88/fTTjBs3rsVxbm5ulJaWAjBq1Cj+/PNP8vLyqK+v54svvmDSpEkAGI3GBunozz//nPHjxxMWFkZSUhJJSUmtGgHQGs989NFHAHz00UfMmzev0f7//Oc/3HHHHRgMBiorKxFCoNPp1Iqgm6g31rNozSIKqwp5fvLzeDhYqNq2M1Bspp9HP8CcOdRB91CO2RDYdjFaQUkaXvVGdCHN3XFhbmFklGVQZ6w7aQjO0IpAGYIuoqysjIULFzb0zt23bx+PPvpow/4FCxYQFhbGoEGDOnTeV199lZSUFB5//HESExNJTExs0Oi35MYbb2TmzJlMmTKFoKAgnn76aaZMmcKQIUMYPnx4wxe2i4sLmzdvJj4+npUrVza0nGzK5Zdfztlnn83BgwcJDQ3lvffeA2DRokWsWLGCmJgYfvvtNxYtWtRwTGZmJps3b24Ijt9+++2MHDmSN998kyuuuKJD163oGt7e/TbrM9ezaPQiYn1iG+/M3AGeEe1uNGPuU9DhFFKjUWu/CDa/IigoPoa3sR6Cm68Iwt3DqTPWkV2eDXb24OR95lJIrUmS9vSf3ihDfeutt8p33323u6chXVxcunsKZ5ye/t44XazPWC8HfzhY3r/6fmk0GpsPeGGwlF9d1aFzTv1qinzglUgp17zQ/oMKjmqyyo+4S/nFFR16vb7Ggs8nyuveiJGyprLZvs1Zm2X8h/FyfcZ6bcNrY6T8/PIufX1akKFWK4IzwPDhw9m1axdXXnlld09FYSNkl2dz3+r76O/Zv1nrSaDDgWIzUZ79OOrg1LEVgdktZHCx+RVBYU2pJjhncGy2z2oK6RlaEfTJYHFPY9u2bd09hQbKysq6ewqK00ytsZZ//PkPquqr+O/k/+JssNIqtYOBYjOR7pH8z7AJWXgM0fZwjRxTxlDYqDPei7dHISUFxhp8XKwXmvo7+2Ovs2+cOZSXbHVsV9MlKwIhxEwhxEEhRIoQYpGV/Q5CiK9M+zcJISJN288RQmwTQuw2PU7tzDykksdVNMHW3hO19bU8uPZBknKTeGzsYw0B3mZk7tAeO7oi8IiiTEBeSUdWBAfAIwy8Im16RVCdn0yZTuDtEWF1v07oCHMLO9mXwDVAqyM4A+/hThsCIYQeeA2YBcQClwshmkSluA4olFJGAy8Az5i25wHnSSkHAwuBT051Ho6OjuTn59vcB1/RMlJK8vPzcXRsvgzvi1TUVnDbytv4+ejP/H3Y35kVNavlwVlJHQoUm2nQHKrIbv8XVM5+8B+kvVZlgRY8tkEK09YD4O0zsMUxYW5hpJVZrAiMtVBZeNrn1hWuoVFAipTyCIAQ4ktgHrDPYsw84FHT798ArwohhJRyh8WYvYCTEMJBStlYQ6EdhIaGkp6e3qJkgsI2cXR0JDQ0tLuncdopqCrg1t9uZV/BPh4f+zjnx5zf+gGZSR12C4FFCqnOyKjyPHD1a/2A+jrIOwj9p2hVxdII1cXg5NXh1+7t5GdqLmJv/7gWx4S5h7EpexNSSoS5ZWVpdocNdkfpCkMQAlgIaZMOjG5pjJSyTghRDPigrQjMXAhsb8kICCFuBG4ECA8Pb7bfYDAQFRV1ipegUPReMssyuWnFTWSVZ/Hi5BeZEj6l9QPMgeLh13T4tfyd/XHS2Z+sJWjLEBQcgfoa8I8Fc8C6osAmDUFBnnZv7O0S0OKYMLcwKusqyavMw8/VVF1clg0BTZ0sXUuPyBoSQsShuYtuammMlPJtKeUIKeUIP7823nwKhY2QXJjMVcuuIr8qn7fPebttIwAnC8naqTFkiU7oiHQN4ajBrn1FZeZAsf8gLS8ebLOozGikoDAVAG/Hlu/uw920m9zjpcfPqMxEVxiCDMAyDB5q2mZ1jBDCDvAA8k3PQ4HvgaullIe7YD4KhU2QlJPEwl80JdgPZ37IsIDmRUpWyUzSHoOGnNLrRnnFcNS+nUVlOfsBoTVhcfbRttliwLjwKAUmZ4ePo0+LwxqlkJ5BmYmuMARbgBghRJQQwh64DFjaZMxStGAwwEXASimlFEJ4Av8DFkkp13XBXBQKm0BKySPrH8Hd3p1PZn/CAK8B7T/4FAPFZqK8Ysiys6Oi8Gjbg3P3g3c/MDiBs8kdVGmDK4LMHRTo9Djq7HGyc2pxWJBrEHqh1zKHHFzB3rV3rAiklHXAbcByYD+wWEq5VwjxuBDCrKP8HuAjhEgB7gbMKaa3AdHAw0KIJNOPPwqFolU2Z2/mSPER/jbkbwS7dlCm+BQDxWbMmUPHitqxgDdnDIFtrwgyd1BgZ8Dbyad5cZ8FBp2BIJcg0kvPbKeyLikok1IuA5Y12fawxe9VwMVWjnsSUC2sFIoO8uWBL/F08GRm1MyOHWgOFI/omPihJQ0ppCXHaVU5q7YK8g9DrEmY0MEddHa2GSPITCLf2QPvVtxCZsLdw0kuStYyh85Q7+IeESxWKBTtJ7s8mz/S/uD8mPNx0Du0fYAlHVQctUaEewQCOFpbDLmHWh6Ynwyy/uSKQAgtYGxrKwKjEbKSKDA44u3UtjtuathUUopS2Hpi68mistOMbRmCVc/A1g+6exYKRaf4+tDXGKWRSwZc0vGDOxkoBnDQOxDiHEiqwQ4O/q/lgWaNIX+L1EdnH9szBPkpUFNGgU62mjFkZl70PLwdvXl/z/ta5tAZMAS2pTV04Cet7VsnlsUKRXdSW1/Lt4e+ZWLoRELdTqFQroOB4nfXHGFfVglBHo4EejgR5O5IoIcjoe79OVyWCwf+B+Pvsn5wzj7QGcDbopmSs/cZqZTtUWTuQAIFdZXtMgSOdo4sGLSAV3a8wsHQ+QysKYPqMi14fJqwLUPg4gvleW2PUyh6EhUFYO8Cdg6sOLaC/Kp8Lh146amdK3OHVS18axRX1PLMLwdwtNNTWVtPnfGkpIS9nwOOPvWUZ2zFpTT7ZM67JTkHwDdG09Y34+x9xoTUegyZOyizd6FW1rXLEABcOvBS3tv9Hh+UJfM0aKsCZQi6CBc/LXjVhJKqWsqr6wjyaDmtS6HoFqSEtyeB3gEu/5IvD35JmFsY40Ja7lbXIhUFWu7/iGvbNfy3/SeorZcsvmkUCaGe5JdVk1VcRVZxFSuOlvFrwUp2Odhz9sFl1s+Zsw9Chjfe5uRte8HizB0UBAwCctptCDwcPLhowEV8tu9TbrfTE1KaDT5d26bWEtuKEbj4NVsRHM0rZ9aLa7jg9fVKsE7R8yjJ0L6885M5+MF0duTs4NKBl57sO9wRjq7WHttZUbxsdxbBHo4khnmi1wn83R0ZEubJzPhAHpvxF0Dwh6MvZTt/aH5wdZmWneTfRBrB2UerI7CVz5qxHrJ3UeCraTS1VkzWlKtir0IIwcfu7qc9hdS2DIGzD9SWQ43WP3dvZjEXv7mezOJKsoqrSM5RWv2KHkb2bu3xgnf4wtsbB6NkfkFOx79IjUZY/Rx4RUHE+DaHl1TVsiY5j1mDg6zmvbvauxLtOYB1jh44pK2lrqKo8YBcUxN2/yYJps7eYKyD6pKOzb+3UpIBtRUUuGvlUe3JGjIT6BLI7PBz+M7NhcKi1NM0QQ3bMgQuJo2iijy2phZw2dsbsdfr+OCakQCsT1Hxg55AZlFld0+h55C1CxCURI1nmYOe2QYfPH5/Er6/WcvTby/7l8KJ3TB5Eejb9giv3J9DTb2R2YOt+P5NjAwcxgnnGqCOVT993ninpcaQJbZWVFZVDEC+yZZ6OXRMbO/aITdRpdPxZc7Grp5ZI2zSEGzbd4gr39uEn6sDX/9tLJMH+hPq5cSGIzby5uzBbE0tYOzTK9mSamN+5JbI3gXe/fgh7Xcq66u4bMbrMPkB2PUlfDgbSrLaPoexHlb9G3wHwOBmdZ1WWbY7i0B3R4aGtfzFNcx/GNWyhu1O3lTt+ZEjuRYr6pz9YOekNaOxpMEQ2EjmkMkQFFAPtC44Z43+XtFMrpF8XpZCRW1Fl0/PjI0ZAl8A3ly2kf5+riy++WxCPLUA8dn9fNh4pACj0UZ8lz2UFfu1nOnNR5UhACB7F8bAwXx18CsS/BKI9YuDyffBpZ9qWTkfzm74smmRPd9B7gFtNaDTt/mSZdV1rDqUy8z4QHS6luUQEv0TAdgbnsBk3Q4e/Gbryc9P7n5NaK7p6zUokNrITVeV5gIrMFbjZu+GQW/o8CmuFV4UyTq+T/m+q2fXgE0Zgh+SawBI9K7nixvH4Ot6sipzbLQPxZW17MuyEd9lD2X1Ic09tzu9jS83W6CyEIqOs9HDh2Mlx7hs4GUn9w06D678BgqPwdLbW44Z1NdpqwH/OIhto1mNiZUHcqipMzJ7cFCr4wJdAglxDWGPqwuuVGJIW8+nm0zS1JYaQ5Y425ohMK0I6iqbBYrTCioorqxt8xRDXcMZWq/j470fU2tse/ypYDOGoKbOyIc7ywG4cbg77o6NLfPZ/bTVwobDNvIG7YHklFaxP6sEnYDdGcoQkL0HgC+qs/B29GZG5IzG+yPGwvRHYN8PsPlt6+fYvRgKDsOU+0HXvo/7z7uz8HNzYHhE2/7sof5D2V6RgdHgwkKvvTzz8wEysjKgNKt1Q2ArCqSmoHhBXXkjt1C9UXL+6+t44PvdbZ/D1Z9rS8rJLM9keery0zJNmzEE9nY63r9hEtLOCUNV86BwoIcj/XxdVJygG1ljWg2cNySYjKJK8ss63LG0b5G9i2SDgT8L93JhzIXY6+2bjxl7BwyYBcsfhPStjffV18Kfz0BgApw1p10vWVFTxx8Hc5gZF4i+FbeQmaH+Q8mvKiC9/wQmswUw8v73P2s7m6aOAjh4gNDb3oqgpqSRIUhKKyKvrIYVe09QXNHGXb5rIBOLcunv0Y8P9nxwWtLcbcYQAHi52CNc/Fp8E47p78PmowXU1dtmc+3uZk1yLj4u9lw6QmvOYfOrgqxdvOXrj5OdE1fFXmV9jBBw/hvgHgRfX9O4WCvpMyhMhSkPnmwT2QarDuZSVdu2W8jM8ACtYGx7QDT68hM8NbKW6gxtJWN1RaDTaW0qbaWorKoYDM4UVBU2MgR/HswBoKbeyE+7M1s/h1sAOuCv/eZTa6wlr7LrsxttyhAAJpkJ6w3ux/b3oay6Tn0BdQNGo2RNch7jY3wZHOoBqDhB8okkfnXUc8WgK/BybMVN4+QFF3+kyRB8f7NWM1BXDX/+B0JGwIAZLR/bhGW7s/BxsWdUVPuyW6I8ovBw8GCHvh50dpyr38pAXTrVehdwD7F+kC0Jz1UVU+foQVF1UaMaglWHchke4UWMvyvfbW/a0LEJpt7Fc7ziWDJvCX7OXd+qVxkCC8b004I561Wc4IyzL6uE/PIaJsb44eZooJ+fC7ts2SDXVvGWzMNJ2LEwdmHb40OGwYx/QfJyWP8SbP8YStJhygPtXg1U1daz8kAOM+Lb5xYCrYfxUL+hbM/fC5HjcT7yC8OcskkhvOXXtSXhuapiihzdkJxUHs0rq2ZXejGTB/hxwbBQth0r5Fh+ecvncNNaVurLck6torwd2KAh8INy61/0vq4ODAxwY6OKE5xxVidrxnlCjBa0TwjxYI8NG4Lkoyv41dmJBQFj8XT0bN9BI6+H+Avh9yfgj6cg/GzoP7Xdr7nqYC4VNfXMjm+fW8jM0IChpJakUhA9FfIOcVbtAXZWB3I0r4UvNxtbERQ4uAAnawhWH9Le65MH+jN/aDBC0PqqwLQiOJ0yEzZoCEwrghYCLmf392FLagHVdfVneGK2zepDuZwV6Ia/uyMA8SEeZBVXkVPagerZPsSbez/ESUquTri+/QcJAee9BN5R2h13B2IDAD/vycLL2cCYfh0rehrmr6mZ7vDSDIhe1nJIhrF8bwtfXM42JDxXXUKhg1arZDYEfxzMxdfVnrhgd4I8nBjb34fvd2S0HAR28QMElOWctmnaniFw9oX6aqgutbr77P4+VNUa2Zlmu3ejZ5ry6jq2HStk0oCTvs+EUE8Am1wVJBcm82vJIRaUV+MZ0MEGMg5ucOW3cME7EDWh3YdV1dbz+/4cZsQFYqfv2NdCrE8s9jp7dpQdbxC0q/U5i1/2tGAIzF3KbEF4rqqYAoOW7eXj6EO9UbImOZeJA/waivXOHxrK8YIKth1rwV2mt9OMQalaEXQdFnpD1hgT5YMQsP6w0h06U2w8kk9tvWSihSGIC3ZHCNidbnsFfm/ufBMXKbjaOarduf+N8IqEhI51L1ubnEdZdR2z2pktZIm93p5433h25OzQ+hPr7OgXN4qktCKyi62s6Jx9wFgLNTYg8lhVTIFJ28nb0ZuktCKKKmqZPNC/YcjM+ECcDHq+29GKe8jt9LastF1D0EKDGg9nA3HB7qqwrAuoN0qW7Mhg6c7W0+NWH8rFyaBnROTJzBgXBzv6+7myO6PoNM+yZ3Go8BC/HvuVK0rL8QxMPGOvu2xPFh5OBsb2b79MsiXDAoaxL38flSOvg5vXMmmoljr66z4rd7G2Ul0sJVSVkK/XoRd63B3c+fNgDjoBE02xMABXBztmxgfy085MqmpbcEm7BqgVQZfiYnqjt5A5BJru0I7jRS3/UxStIqXklz3ZzHxxNX//Kom/f7mj1VTQ1cl5jOnnjYNdY12ahBAPdtlYCumbO9/Exc6JqwsLtEKw00RVbT1JaUV8uvEY93+3m593Z3NubACGDrqFzAz1H0qdrGN3wQHwH0S0vyvR/q7W3UO2okBaWwnGWgow4uXohU7oWHUol8QwTzydGxcHnj80hJKqOlYeaCEO4Hp6exfboCEwrwhaNgRj+/tSU29s2WensIqUmv9z3mvruPnTbRil5IVLh+Dj6sCi73ZZLdRLK6jgaF45E2Ka50YPDvUgp7SaEyW9L2CclFbEa3+kdKgK9FDhIVYcW8EVviPwNBohqGsNQU5JFf/8ZifnvvAncY8sZ/5r63hoyR6W7c5ieIQXt0yJPuVzD/EbgkCwPWd7w7YZcQFsOlpAYXlN48ENwnN9/PNllpdAa1HZkDZq4RYyMy7aF383h5azh9wCtGCx8fTcnNpWq0rQgsXQau/ikVHe6HWC9YfzGBft2+I4xUn2Zhbz5E/72XAknxBPJ569KIELhoZgp9dhr9dz6+fb+WBdKjdM7NfoOHPaqDk+sCNnB6/ueJXHxz3O4JCThWUBsY5n9oI6yet/pPDrvhP4uTlwialSui3e3PkmLgYXFuIBOjvwO6vL5lNQXsOCdzeRVljB2P6+zIwLJDbYg7hgd0K9nKw2n+kIHg4eRHtFa3ECEzPjgnjtj8P8tv8EF1v+DWxlRWCWlzDW4O3o35A2OsWKIdDrBOcPDeG9tUfJL6vGx0IQE9BWBLJe+5u5Nj++s3TJikAIMVMIcVAIkSKEWGRlv4MQ4ivT/k1CiEiLffebth8UQrS/BPJUMTiCg3urhsDVwY6EUA8VJ+gAN3+6jYMnSnnkvFhW3juJS0aENWSfzB4cyLSz/Hl+xSHSCkya6ru/gaeCiF5zF3Pdkunv68TmrM3ctOImNmdv5rvk74gNdkcn4OCxNNjxGXxzLWRs68arbB919caG984TP+5rV6OdjLIMbTVw1hV45BwEv0Fg59Dmce2hpKqWq9/fxPGCCj64ZhTvXzOSu88dyMz4QMK8nTttBMwM8x9GUk4SdcY6AOJD3AnxdGqeRmorwnNmQ1BfibejN6ss0katcf6wEOqMkp92WekxYSoqqypsQ47iFOm0IRBC6IHXgFlALHC5EKKp2tR1QKGUMhp4AXjGdGwscBkQB8wEXjed7/Ti7NOqawi0OMHO9GLKqutO+3R6O2XVdaQVVHLd+Cj+Oi6qma9fCMHj8+MRAv7vhz2au2T/UqTQM6h0Iy/XPsK61xO5ZcWNhDj5M9h3ML8e/RmnvV/xhcvz3LRpBvxwC+z5FnZ+2U1X2X52phdRWl3HollnUS8l9327q00X0ep0rZ/w3P7nac1ousgtVFFTx7UfbOFgdilvXjmcs08xGNwehvoPpaKuguTCZED7v8+IC2S1KSOpAUdPEDobWBGYXEO15Xg5eLO6SdpoU84KdCc2yJ3vtqc32i6lZE+JVovwjw9+PS2xy65YEYwCUqSUR6SUNcCXwLwmY+YBH5l+/waYJrTbkHnAl1LKainlUSDFdL7Ti4tfi+mjZsb296XeKFWnrHZg7kzV38+1xTEhnk7cc+5AVh3M5aedmXBsPflh0xlZ/RrvDvkbd7hBVFUl7+9ZzwVZh0ktTePgz3cSI9L4XMxGXv87hI6EE3vP1GWdMqsP5SEEXDYyjAdmD2JNch6fbTre6jFr0tcQ5hZGhHDUblICB3d6HlW19dz48Ta2Hy/kpcuGMuWsrncpWGIuLGsaJ6ipM/LnQYsbL1sRnqsqokoIyuurqK5xbpY2ao0LhoWwM72YlJwyKmrq+HzTcWa9tIabf9BiBzMiBNV1XS+K2RWGIARIs3iebtpmdYyUsg4oBnzaeSwAQogbhRBbhRBbc3Nbv5tvExe/Vl1DAMMjvDDohXIPtYMjuZqUQH8/l1bHXTM2koRQDz768Vcoz2U7sRjdD/Bq6XLO8h3Mu/O+w2vCP5iGK3oEy8dez4+Tl/Nw5WVku8VBQDyc2NPjC5HWJOeSEKplhiwYHc6EGF/+tWw/x/OttxqsqqtiS/YWxoeMR5ib1XcyY6i23shtn+9gbUoez140pN1qop0hyDWIQJfARnGCEZHe+LjY80tT95C5qKwvU1VMkakOJDNf1yxt1BpzE4PRCbh7cRJj/vU7D3y/GyEEd84bD8CcKIGHU8e7nLVFr8kaklK+LaUcIaUc4efXSfU9l7ZdQ072eoaGeylD0A4O55ahExDu49zqOL1O8K/zB3NW9S4A3igrxTH4C4b4JfD2OW/jERAHU+7H6/rfGR18NssLdhFvUiLdlV4MAXGa37WkDbXGbqS4spaktKKGD7wQgmcuTEAvBP/4ZqfVVqhbT2ylqr6KCSETIHuntjEw/pTnUG+U3LN4J7/tP8Hj8+K4aHjoKZ+rowz1H8qOEzsaXGF6neDcuABW7j/R2KXh7NP3YwTVJZSYDEFKdr3VtNGm+Ls5Mm1QAHszS5gwwI/FN53NsjvGc/GYGHD0OG0ppF1hCDIAy7SIUNM2q2OEEHaAB5DfzmO7jHd2vcPSw0upd/bV7kaMrS+xzu7nw57MYtUgpQ2O5JYT7u3cLDZgjfgQDy73T+MTFz+SnX4g2DGeN6a/gat9Y7fSzMiZpJWmoXPMQK8TWh1CgOnL8cS+03EZXcKGw/kYJY3SYYM9nfi/82LZdLSAD9enNjtmbcZaHPQOjAwcCdm7tcpgR49TnsOrK1NYujOTf84cyNVnR57yeU6F0YGjyanMISk3qWHbjLhAymvqG1fr24LeUFUxZXba3fuxXGObbiEzL12WyJYHp/PaFcMYFeV9Mph/GovKusIQbAFihBBRQgh7tODv0iZjlgJmLd2LgJVSu2VYClxmyiqKAmKAzV0wp2bUG+tZlb6KB9c+yEW5K1npaEC2IYU7e3AQUsKSpNMTqe8rHM4to18r8YFGSEkWe/iPnxP15f15bPTzOBuarySmhk/FTtjxR/oKYvxdtR4RAaYchBN7unD2Xcua5Fxc7PUMDfdstP3i4aFMO8ufZ5cfaIipNByTvoaRgSNxtHOErF2dcgulF1bw+qoU5iQEccvkU68LOFVmRc3C08GT93a/17BtbH9f3BzsWL7H4m7W2RZcQyWUOmifC2O9I5MHts+T4Wxvh7eLlZWD6+mTmei0ITD5/G8DlgP7gcVSyr1CiMeFEHNNw94DfIQQKcDdwCLTsXuBxcA+4BfgVinlaamY0Ov0fDLrE56b9Bx1QnBngB9X/nYjW7K3tHjMwEA3hoR68PXWtNPSHq4vUG+UHM0rbzM+YGbjoe/5p7sdA+z9Odd3ESMjAqyO83DwYEzwGH5N/ZXBIe7szihGOriDR1iPDhivSc7j7P4+zSp0hRD8+4LBONjpuefrndSbXETHSo5xvPQ440PGa1kmhUcbGQIpJU//fIBlu62kFFrhX8v2IwQ8MNtKd7AzgLPBmQWDFvBn+p8cKjwEaG1ipw7yZ8X+EyeLCp19tBVBX/5cVRVTaq9l+3g6uhEffOqrPADcAnv0igAp5TIp5QApZX8p5VOmbQ9LKZeafq+SUl4spYyWUo6SUh6xOPYp03EDpZQ/d8V8WkIndMyInMH3Ix7i0dx8sityuHb5tdz8280NKW9NuXhEGAeyS1XXshbILKqkus7YrhXB7tzd3LH5SSJqa3lv0jO8dOnoVhugzIicQUZZBn6+ORSU15BRVKnFCXqoITiWX87xggqrVdIA/u6OPDY3jh3Hi1i8VcuRWJuxFkCLD5hXOhapo7/vz+HNPw/z96+S2JvZ+ntw/eE8lu3O5tbJ0QR7OnXBFZ0al591Oc52zo1WBTPjAikor2H78SJtg5O3pgJc00pDlt6OhSEYGxnaYtpouzGvCFTP4q7BzjWQC8vK+d/gO7ln+D3szt3Ndcuvo6quuZTBeUOCcbDTNXxwbZraKvj1oUZfxIfbkToKcLjoMH/7/W94Sx1vF9fhETy8zZebGj4VO50deVJbte3JMAWM85O1Vow9jDXJmg98QiuZIfMSgxkV6c1/fz1IaVUtazLWEOEeQbh7uOYWgobU0bp6I//+eT+RPs54ORu47fMdLda11NUbeWzpPkK9nJpVb59pPBw8uGTgJfyS+gtppdrnxtz6cld6kTbIXF3clwPG1SUU6rQYwdio9lWXt4pbIEhjixL6ncEmDQEu2gfVsbKIa+Kv4cUpL1JYXcgvqb80G+rhZGBWfCA/JLWiDGgrrPg/WP8KfHF5Q9XkYVPqaL9WXEMZZRnc+OuNGHQG3imswi9sbLsaprjbuzMueBzb81dhp7PIHDLWQd6hrrmmLmRNci4hnk5E+bb8txBC8NCcQeSV1fDSyn1sydqirQZACxQ7+4Kblur51dY0DueWc//sQbx02VCO5Zfz4Pe7rbopP998nIMnSnnoL4NwNJz+msy2uCr2KvRCz4d7PgTAx9WBQHfHk/0lbEGBtKqYQqFHGu2I9LFeTdwhRt8MD+WAYxecqwm2aQjMdyOmWoIRASOI9ozm8/2fW/2QXTIijNKqupY7LtkCB3+GzW/DgJlQnA5LbwcpOZJbhoeTAR9rwS0grzKPG3+9kcr6St4a/ShhhWkQOb7dLzsjcgYnKrKJDMnT3HP+cdqOHuYeqqs3sv5wPhNifFuXbMjaRcKSc1jq9wYndv6bGmMNEwJHa/uyd2puISEoq67jhRWHGBnpxbmxAYzp58Pfpw/gh6RMvtrSeHVaWF7Df389xLhoH2bEBZ7Gq2w//s7+zO0/lyUpS8ir1D5nccHu7M009Zdo0BvqwyuCqmIK0SGNjgR6dIFWlt7QoY5zHcE2DYHeoFU2mgyBEILLBl7G/oL97Mrb1Wz4mH4+hHk72a57qCQLltyiuSwu+RimPQz7foCt75syhlysfvmty1jHZT9dRm5lLq9Pe50BZp2UiHHtfukpYVOw19nj7LWHXenFSJ/+oLfvcZlDO9OLKa2qazE+0MCur6DgKLG6NAJdN+JkNDL8w4vgnWmQs7/BLfT2n4fJK6vhgdmDGv62t06JZly0D48s3cvB7JPugf+uOEhZdR2PnBfXZbpBXcG18ddSJ+v4eN/HAMSFeHA4t4zKmnoLBdK+bAhKKAJkvRNBHt0Xs2kPtmkIQFuCWxSVndf/PFwNrnxx4ItmQ3U6wcXDw1iXkn9SNM1WMBphyc2atvqF72tCaGPvgOjp8Mv96HP2NosPlNWU8ej6R7n5t5txMbjw/oz3SfRPhGNrNQPs31SKqmVc7V0ZFzKOPLmF4spq0otrNVXOHrYiWJOcixAwLroNLZ+U3yByPPq/J7HMsx8OFSEUxF6rGTcHN4iezomSKt5Zc5Q5CUEMDT/ZrEevE7x46VDcHA3c+vl2Kmrq2JdZwuebjnPVmAgGBLid5qvsGOHu4ZwbcS6LDy6mpKaEuGB3jBL2Z5f0/RhBfS3UllMsJXqccLLvfndda9iuIWgiM+FscGZe9DyWpy5vWMpacuHwUISAb7alN9vXp1n/MhxZBbOeBr8B2jadDs5/C6OTF4/XPMdAL4vhmes5f+n5fJ/yPdfGX8vi8xYT72sqBEtdB+FjO9x+cUbkDErr8tE7HTfFCeJPS1FZWkEFV7+/udHddntZk5xHQohH65WjRWmQewCip5NakkqJMZfKunHcmDUX4zXL4L5UiJrI878eos5o5J8zmstQ+7k58NJliRzOLeP/luzlsR/34uFk4K7pAzo85zPBdYOvo7y2nK8OfNWgurk3swScPAHRd2MEpoBuiazHQde+1OruxIYNgW8z4blLB15KnbGObw9922x4iKcT46N9+WZbulWZgD5JxjZY+QQMmgvDFjbe5+LLkYkvESmy+cvx/1BWXcpjGx7jphU34WTnxMezPuau4XfhoDdJKZdkajnyEWM7PI3JYZNx0Dtg77mb7ccLtYBxWXabelEdwWjUVEJXH8rlxd86FoguqdJkJdp0C6X8pj3GnMOa9DUA3DLmL+zOKOZ7U7/aA9klLN6WxsKzI1uU7BgX7cvtU6L5dns6m44WcO+MgXg4d73+TFdwlvdZjA8Zz6f7P8XbFTydDezLLAadXjMGfdUQVBUBUE49znbKEPRcXHyb6Q1FeUQxNngsiw8tptZY2+yQS0aEkVFUyTpbaGxfXQrfXq81xJj7crMglZSS3+vduMl1HP+u2sA5iyfzXfJ3/DXuryyes5ghfkMany91nfYY2f74gBkXgwsTQibg6LGHTUfzLCqMu8499Pnm46w/nM9ZgW78sjebY/ntz2/fcDifeqNsNW0U0AyBRxj4DmBtxlqiPKK4ZuRwhoR68OzyA1TU1PHvZQdwc7DjtqmtVwXfOX0Akwb4MSLCi8tGhrd7rt3B9YOvp6CqgCUpS4gLdmdPhkXAuK/GCExZdZXU4mbfs1x21rBhQ+CnvQmbtH67bOBl5FTk8MfxP5odck5sAB5OBhZvtQH30LJ/QmEqXPC25tc38fux31m0ZhFTv57Kq4duZpP/cQ47uzOzpJhPxjzJ3SPu1qQSmnJsrdYQ6BTlE2ZEzqBOFHOgcCdlngO1jV1kCNIKKvj3sv2Mj/blo2tHYacTvLf2aLuPX5Oci7NJpLBF6mo0F1v0dCrqKtl6YisTQiag0wn+b04sJ0qqufnT7fx5KJfbp8a0KU6m1wk+/OtIvrrp7FaL8noCwwOGM9R/KB/u/ZBBQS4czC6ltt7YtxVITb0IanW1eDp0fbpnV2PbhgDZ7I5kYuhEgl2CrQaNHQ165icGs3xvNkUVNc329xmSPoedn8OEexvdwa/LWMffV/2dDZkbGBk4khjdX/HKf5RlF/7MIxWQsPwxqCyyfs7UdRA+RnMJnAITQyfiqHfGzmsdW/IM2v8vp/OGQJoaxwghePrCwQS4OzI/MYTFW9Oa99ptgbXJeZzdzwd7u1Y+TmmboKYMoqezOXsztcZaTVYCTar5LwlBrD6US6iXE1ePjWjX6woherwRMHNd/HVklWeByx5q6o2k5JT1bQXSqmJqAamrx8e5k9ISZwDbNQQNtQSN3UN6nZ5Lz7qUrSe2WpWduHhEGDV1Rpbu7EVCdL89Cp9domX+tEXGNvjx7xA5ASbd17C5zljHc1ufI8wtjBUXreDZic9SnjeCGO9wreLxko+1GMC31zVvsF2Wo1UDdyBttCnOBmeuGnQ1Bve9/HxoS5dJTXy2SXMJPTB7EKFemk/+hon9qKo18unGY20efzy/gtT8ina4hVaAzgD9JrEmfQ1Odk4MDzhZXb1o5ln093Ph0fPi2qXi2tsYHzKeAOcAUipWAaaAcV9WIK0qptSUFOHv6tm9c2kHtmsIXEyBPSt9CS6IvgAHvQNfHmjeFjE+RGv43atqCo6ugeTlsPhqzUXREmU58NVVmqbJxR+C3q5h1/cp35NSlMLdw+/GXm+vic3ll9Pf35Q6GjEWZj+n+cF/e6TxeY+Z4gOdMAQA1w5eiE668GfOx1rmUM7+5kanA1i6hC4fdVICYECAG5MH+vHRhtQ2q8nXpGjvnwkD2goU/w7hY5D2rqzNWMuYoDHY60+6f8K8nfn9nslMj7Uuwtfb0ev0nNf/PJLyNuHkWK5VGPdlQ1Bd0mAIgt1acRn2EJQhsNKy0tPRk1lRs/jxyI+U1JQ023/JiDD2ZJScLJfv6VTkaUHf5F/h+5usf3nW18LihdoH87JPG2Q4QKsLeHXHqwzzH8a08GkAZBRWUlNnpJ+lnMKIv8LIGzQZCsvewqnrwOACwYmdugxXe1cS3S6kwm4f6x2doa4KCo60fWBTjEZk/hEWfacVDz594eBmhVg3TuxHXlkNS3a03B7DaJT8b1cWwR6Ojf8OTSnJ1ArgoqdzpPgImeWZDW4hW2Ju/7nUy3oCg/exL7NEixHUVUJNH6zNqSqmzGQIQjyUIei5mL/oWkhBvPysy6msq2RpStPWCjA/MQQ3Bzte/t26YmmPo6IA4ubD9Mdg73fwv7ubKxj+cj8cXw9zX4Ggxhk/7+15j4KqAv458p8NX5iH80xic/5NxOZm/ltzKy29A9K3aduOrYOwUVpFdye5IvYyjLXu/PfEZiScmnto81vIV0eQkpLMA3856RKy5Ox+PsSHuPPOmiNW04WllDy8dA/rD+dz/YR+rVf0pvyuPcacw9bsrdr5g8/u+Lx7OVEeUST4JlDjtJl9WcUYnfpwUVlVCXl22vvK19mze+fSDmzXEDh5gdC12LIy1ieWIX5D+PLglxhl405mHs4GbpjYj1/3nWBnWtEZmGwnqKuG6hKtknr832H83bDtQy1uYGbHp7DlHTj7Nki4uNHhmWWZfLz3Y87rdx5xvnEN2w/naIag2Z2w3gAXf6TFDb68Qvuiztl3Smmj1hgbFUht/jQOVR5hjbNzxw2BlNRuehedrOfikHyuGGU99VIIwQ0T+nE4t5yVB3Ka7f/vr4f4dONxbprUj2vHR7X+mikrwC0Y/GPZV7APTwdPQl3PXPvInsTc/nMpMaZRwXFyjab3Tl/MHKoqJl+nZc+5GtrZtKkbsV1DoNNrAeNWehdfOehKjpUc45ejzVVJ/zouEi9nA/9d0b7io6+2HOfndjYX6VLMHzKz2uO0h2HEdbDuRVjzPKRvhZ/ugqhJ2oqhCS9ufxEhBHcMu6PR9iN55Xg6G6x3UnLxgcu/1LJkPvyLti2ia1whbo4GBjhPw2D05RVfP4zZHdQcOrYeQ2EKANcPqGz1Tn724CBCPJ14e01j99M7q4/w6h8pXD4qjEUzm1f/NqK+Dg6vguhpIAR78/YS6xPbozSBziQzo2ZiJwwYPLdzuMz03umjhiBXpxVTutur9NGeTROZiaacG3kuA7wG8MqOV6itb1xg5uZo4G+T+7P6UC6bj7a+tN14JJ9F3+3mpS5yJeWXVbe/Y5r5Q2Z2hQmhBXUHXwy/PwafXKDdvTcJDgPsyt3Fz0d/ZmHcQgJdGqtaHs4po5+vdbE5QCv6uuBtqCwEO0cIGdaBK2ydMVF+VORM54Be8mthxwxB5cb3KJHOFBv88Cxp3Ygb9Dr+Oi6SzUcLSDKt/BZvSeOpZfv5y+AgnpzfPLbQjPQtUF0M0dOpqqsipSiFOJ+41o/pw3g4eDApbBIG9yR2F5n+dn0xYFxdQj6aK7RpP+6eiG0bAmefVg2BTui4c9idpJel813yd832XzUmEn83B55bfrDFL+biylruWbwTKSE5p6zTPQ0OnShl9L9+Z/nedvYuNV+fs0V6o04H89+AgbM1bf/LPj+5YjAhpeQ/W/6Dr5Mv18Vf1+y0R/LK22xGw1l/gb/8Fybco4nVdRGj+/lQVZhAhM6d1+xrqWuj93QDFQUYDv7ID8bx2IcNa5db6bJR4bg52vHOmiP8vDuLRd/tYkKMLy9cmti+HP6U30Dood9kDhUeol7WE+vTftG9vsj50fMRduWsLDAZ4j5oCIyVReQJPSBwMSiJiZ6Ni1+rriHQWggO8x/Gm7vepKK2cXaDk72e26ZGszm1oKE7VVMe+WEP2SVV3DypP/VGyYFTEDSz5OMNqdQZJRuPtHM5bVoR1Do2/qJHb4BLP4O79zVIH1vy67FfScpN4vahtzdrLl9SVUtuaXX7GtaPvB4m/bN9c20nIyO9EELHBLuJpNob+HHX++06rnzLZ9jJGk7EXIZT6BCttqG2eVc6S1wd7FgwOoKfd2dx55dJDA334q2rhrdePGZJygotUO7kyd58zfDY8ooAYGzIWOxx50CtFjjvi8FiY2URJTo9DjpndKLnf832/BmeTlz8rKaPWiKE4K7hd5FXmcdn+z9rtv+ykeGEeDrx3K/NVwVLd2ayJCmTO6bGcOUYLSjZmZTT0qpavt+upTPuaG+Q2rQimPHWHj5Yd5SaOovAt05nUoFsTHV9NS9se4EBXgOY139es/1H2tGV7HTi6WzPwAA3MorHEF9dzRuHv6Wmvo0qYCmp3PgeScb+zJ95rlaQJo2aGmgbXDM2Ejudjn5+Lry/cCTO9nZtHgNodRlZOzXJbmBf/j68Hb2budlsDYPOwGCPKdQ77afA0bOPxgi0OgKXXhAoBps3BL6aOFRrRVZAon8ik8Mm8/6e9ykyqQqasbfTcef0GHalF/PrvpPumsyiSh76fjdDwz25dUp/Qjyd8HQ2tNmAvDWW7MigvKaeMf282Z9ZQnVd224mWZ6HEUG+0ZXHftzHuS/8yc+7s1qNMby24zUyyjK4d8S96K1IQhxpZ5/i08noKG9WZNhzR2kNWbWlVo20JeWH1+NbeZTdgecT7e+mFaRBu9xDgR6OLLtzAl/ffHbHVD7NaaMmQ7A3fy+DfAbZbKDYkrnRcxGinh9d3fuea8hoRF9TSrlO4N4LBOdAGQLtsY1VAcAdQ++gvLac9/c0d0NcMDSEfr4uPP/rIYxGidEouWfxTuqMkhcuScROr0MIQXywx0nlxQ4ipeTTjccZHOLB1WdHUlNv5EBW226m4vxsCqUrD50Xzwd/HYm9nY6/fbadi97cwLZjzX3r6zLW8cHeD7hkwCUt5rofzi1DrxOEe1uXST4TjO7nQ2WtkXinfkyUDjy/7Xne3f1uiwYubcUblEonhs0yxTu8o8DOqd3pp9H+rrg5drAOImUFuPhDYAKVdZUcKTpi824hMzNihlFfFcRSR13HVwSZO04a2Z5ITRkCIxU6iddp6C98OrBxQ2CWmWjbEMR4xXBe//P4/MDnZJc37l1sp9fx93MGcPBEKT/uyuS9tUfZcCSfR86LJdIizz4uxJ2D2aWN3TPtZEtqIQdPlHLlmHASwzwBGjJZWqMkP5sC6c6ISG+mDPRn2R0TePqCwRwvqODCN9az6NuTrTnzKvN4YO0DRHtG84+R/2jxnEdyy4nwdm6/n/w0MCpKi3kcs+vH81k5zIqcyUvbX+LRDY82kxCvKi0g8sRytrhNJS4qWNuo04P/WV0iXGcVYz0cXqmtBnQ6DhYcVIFiC1wc7HCvG8Mhg5HDVa3H6Rqor4U//gXvTIUvF2ipuT2Rau1mr0YPHr1AeRQ6aQiEEN5CiBVCiGTTo9VaaiHEQtOYZCHEQtM2ZyHE/4QQB4QQe4UQT3dmLqeEOZOmpYBxyu/w+tgGSdlbEm+hXtbz5s43mw2dMziIswLdePrnA/xn+UHOjQ3gkhFhjcbEB3tQU28kOafjAeNPNh7DzdGOuUNCCPJwxM/NoV3FbLWlOZTq3Ik0NTmx0+u4bFQ4q+6dzMXDQ/lySxrZxVUYpZEH1z5IeW05z0581rqUtAlzn+LuxNfVgf5+LmyrCsKhupinE27jxoQb+S75O/72298aSYPsXPY2jtTgN+mmxifpIuE6q2Rs11JnozVJjn35Wkc1tSI4yRCvqegk/GBsh7s0LwXeOxf+fEZrdVpXqQX7eyKmXgR1dvW9ohcBdH5FsAj4XUoZA/xuet4IIYQ38AgwGhgFPGJhMJ6TUp4FDAXGCSFmdXI+HaOtFcGW97Q7RlNnqRDXEC4deCnfp3zPkeLGRUY6neDucwaQVVyFu5OBf1/QPMc8PkSTo93bQfdQbmk1v+zJ4uLhYTjZ6xFCMCTUk6T0ojaP1VcVIJ19m83FxcGOK0ZrAeyktEI+3vsx6zPX88+R/yTGK6bF89UbJal5Fd0aHzAzup8PK/K1/6EuZz+3D72dJ8c9ybYT27hq2VWkl6ZTW1eP94EvOGwXQ/yIiY1PEBCv3QSUNa8cPiXqajSBvxWPaJpOQgf9pwJafMDb0ZsA574pKncqDA0JI6Lcjf/ZG6lvSTxQStj6Prw1QVO3vfgjuPBdbV/WLuvHdDcNvQjqekUNAXTeEMwDPjL9/hEw38qYGcAKKWWBlLIQWAHMlFJWSCn/AJBS1gDbgTNbd99ajKCq5GRrwYM/N2y+YfANOOodeXXHq80OOSc2gLvPGcDbVw/Hx7V53nyEtzOuDnbs6WDAePHWNGrrJQvGnJRDGBruyZHccoorm3dSM5NXVo1rfTGOnv5W9w8KcsegF/x+ZBsvbX+J6eHTuXjAxVbHmkkvrKCm3tjtKwLQAsZJ1UHakxNaYdm86Hm8Nf0tcitzWbBsAR//+g4xMpW6xKuaB2kD4hode0qU5cKWd+GLy+HZKPhoDmx4DdyD4YJ3Guoz9uXvI84nTgWKLYgLdiesJIgcvZ5fDjfX9KIsB764TKt8Dx8Df9ugaWb5xGhFitk91RAUI4FaUWMzK4IAKaVZNyEbsHa7EwJYajanm7Y1IITwBM5DW1VYRQhxoxBiqxBia25uO32KbeHooWnEW3MNHVoO9dXgO1BT7TT5I32cfFgYt5AVx1awI2dH0zlyx7QYhrXQqUqnE8QGu3cohbTeKPls4zHGRfs0ugsfEuoJwK5WVgXbU/PxohQv3yCr+x0Nes4KMrCy4Hl8nX15dOyjbX5RmVNHe8SKIMqHMpwpcQzRVm9rX4TSE4wKGsWnsz/F2c6Z10+8TpK9KwOmX9P8BP5mQ9AJ99C318L/7tHOkXApXPYF3HcUrvkJBl8EQEVtBUeKj6j4QBPigj1wKQ8joraWResf5s6Vd55caR/fCG+O17q6zXoWFnwL7qb3sd5Ocw9l7ey2ubdGTXkhFUIgkbgZ+oghEEL8JoTYY+WnUYK51NI1OtzVXQhhB3wBvCylbFFTWEr5tpRyhJRyhJ9fG9rv7X9xq72LAdi3BNyCYMoDWiPqtI0NuxbGLSTENYT719xPaU3H/P3xwR7syyqh3oqipTVWHsghs7iKq8Y07lo1OFRzM7UWJ9h35Bh6IfELCLa6X0pJvfe3VJPHv8c/jYdD252UDptSR9tVTHaaCfRwJMLHmbe97gLPcK0PwvOD4Isr6Jd9iMv87sfLWMedwQFk11mROnbx0f7HnTEEeckw+BK4cyfMeR7Omg0OjT/8BwsPYpRGFR9ogreLPXpHPxZnZHN79CVsyt7EBT9cwBNLryDv47lg7wI3/AGjb9JqXiwJStBWBO2VWjmDlBbnN/Qi6DMrAinldCllvJWfH4ATQoggANOjNWdrBmAZNQ01bTPzNpAspXzxlK+iM7j4No8RVJdC8gqInadlfejtG7mHXAwuPDPxGbLLs3li4xPt1/0B4kPcqao1NuTit8UnG48R4O7A9EGNF1seTgb6+7mQlNby6uLIseMAGNysu4aWH1tOWu06avKm4caAds3ncG45Xi2JzXUDoyK9+fREBMa//gK3boGzb6Xu+Cb44lIu+v0i3sjOocZgxy2/32K1twT+safuGqqtgtIs8InWbipaoCFQ7KsMQVO8fANxlpIbgybwv/O+4xJDAN8V7GJ2WCBvjL2aCu9I6wcGJmhB2aLjZ3S+7aG8uKDBENhKjGApsND0+0LgBytjlgPnCiG8TEHic03bEEI8CXgAf+/kPE4dZyuGwOwWip0PDq4QNREO/K/R3ccQvyHckngLPx/9maXW/JstYA4YtydOkJpXzupDuVw+Khw7ffN/1ZAwT5LSiqwaouq6evJOmOyti4/V8y8+uJhg5zBq8qa2W05byxjqOW/u0f18KKqo5VBOKTmO4TxQfglxxS9wh7yXHJ9R9IuezYtTXia1JJW///H35hXIAXGQe1BLTewo5i8hr9Z7DO/N24uvky/+ztYNsi0TGKStVmsyduHz5ZU8cHAjS4LmMD58Oq/vfZ+FvyykrMbKTZO5Z0b27jM42/ZRXVZAvtBqTvrMiqANngbOEUIkA9NNzxFCjBBCvAsgpSwAngC2mH4el1IWCCFCgQeBWGC7ECJJCHF9J+fTcazpDe39XnMZhI3Wng+cpWUs5DVWq7wu/jpGBIzgqU1Pcayk7f62oOn3Oxp07Sos+2LTEfQ6weUtaOYnhnmSV1ZNZnFzvZy9mSW4mdPyLAXnTJwoP8HW7K3Mi56Du6N9uyUrjuSW078HBIrNjDbVEzy2dB+T/7OKxVvSuHxMfx75xz+JuuMn9Jd+zOjgMTw+9nG2ZG/h4fUPNzacAfFQXwP5hzv+4kWm/7lXZKvDzIFiRXMiwrT3tv3v/6cZ5Es/JWLG0zw/5XlemfoKyYXJ3LXqrmbqv/jHallZPTBgXFNeRI5OS9fuMzGC1pBS5kspp0kpY0wupALT9q1Syustxr0vpYw2/Xxg2pYupRRSykFSykTTz7udu5xToKkUdXWZli00aO5Jv+QAU1brwWWNDtXr9Px7wr8x6Azct/q+5m9WK9jpdQwKajtgXJP8B3/fMo0FMXUEuFvP6TcXllm7m9+WWoiPMMUvnJuvCJanLkcimdVvFkPCPNu1IigoryGvrJ1ic2eIUC8nQjyd2HAkn0kD/Fhx9yQenRvXLGvrvP7nccfQO/jfkf/xyo5XTu7oTOZQYar26NnyikAFilsnJiKMammg2DkCblgJg85r2Dc5bDKPjn2UjVkb+b/1/9e4QZS9M/gO6JEppLKymHy99v6zlRVB78fFB2rLT/ZNPfSL1gs3bv7JMR4h2lL0YPMGNYEugTw+9nH25u/llaRXmu23RnywB/syS6y2QDRzcNsqnKjmRrf1LY45K9Ade73OuiE4VkiUs2ml4NJ8RfBL6i8M8h5ElEcUiWGeHDxRSmVN69pFqw9pKyfzXXhPQAjB21cP54dbx/HGlcOJaqV38PWDr+fCmAt5Z/c7fLb/M0prSpE+MaCzO7WAcWGqlsbo2nJtwIGCA0ikWhG0QJC3Gzc5P8c9ni+B38Bm++dHz28w4C9tf6nxzsCEHrkioLqEYlNBpq3ECHo/TZvY71uiNXoPG9N43MDZkLbJavHZtIhpXDzgYj7Y8wEbMje0+ZLxIe6UVtdxvKDlpt1ZqZoqZsjxpdabzaMJ3sUGuzdz60gp2XqskAGu1WDv1qwXQFpJGrvzdjMzaiagrSzqjZLdbaxSVuw/ga+rQ0Pqak8hLtiDIabVUWsIIXhozENMCJnA05ufZuwXYxn+5dlMDwvh0oyf+Ntvf+PhdQ+z4tiKttVMQXMNeYY3z2ixwCw9rVYE1hFCkDD0bFYereBEiXVJ8OsHX8+lAy/l/T3vNxYXDBwMJRlQ3rPUSw21JZTaackUakXQW2ioLs7V3ELJKyB2bvMP94CZgNQCyVb4x8h/0M+jHw+sfYCCqtbVFOOCWw8Y780sxqk8HaPQI0oytFzqFkgM82R3ejF19SeXzWkFleSVVRPuWGE1UPxLqraymRWpubyGtOJiMlNTZ2T1wVymneWPrj0NWXoodjo7np/8PM9OfJZ7R9zLlbFXMsbeF5+aSgqrCvkj7Q/uXnU3U7+eypMbn2RX7q6Ws8IKU1t1C4EWH/B38sfPuYtSnvsg84aGYJTw485Mq/uFENw/6n6mhk3lmc3P8Gvqr9qOoATtMbtn1RM41pVRYTBgr7PHQd91DZlOJ8oQNOgN5UPycs0tFDu/+bigIVoD8iZxAjNOdk48O/FZSqpLeGLDE62+5IAANwx60WLA+PNNx4nQ5VAXPRMcPSHp8xbPlRjmSWVtPSkW6ajbjmuGyE9fajU+sOzoMob6DyXIVSvQ8XV1INTLqVURu81HCyitrmN6bO+XSHC0c2RW1CwWxi3k7uF382TEebyekcaX097gj0v+4M3pbzI2eCxLUpawYNkC5i6Zyzu73qGwqolaa+HxNgPFe/P3EuurVgOt0d/PlYRQD5YkZbQ4Rq/T88zEZxjiN4T719zP1uytmmsIelScoLquHmdZToXBrtesBkAZgpP+8/JcLVvINUArZ2+KEFr20OGVLXa1Gug9kFsSb+G347+xPLXJyqGuuiFF0d5Ox8BAN6u9Ccqq6/hxx3FCRD72gWdpvYUP/ASVRVZf03w3n3T85P6tqYW4OtjhUlfcLGMouTCZlKIUZkbObHae1gzBb/tP4GCnY3x083hDr6ehN8E+7HR2jAsZx7MTn+WPS/7gsbGP4e3ozcs7XmbeknksO7JMWyFUFmq9iFtJHS2vLSe1OFW5hdrB/MQQ9mSUkNKKIKOjnSOvTH2FYNdg7l51N8V6PXiE9ag4wYniatypoMpOrwxBr8LsGio6drKIzEozFkCLE9RWQOqaFk+3MG4hcT5x/GvTv066iEqz4Y2x8N0NDeO03gTFzdwOPyRl4F6bi556ze2QeIW2StnbvGcyQKSPMx5OBnZaSE1sO1bI0HBPREV+s0Dxz0d/Rid0nBt5bqPtQ8M8ySiqJLe0utlrSCn5bf8Jxkf74mTfwt+mN2POHMrZ12izm70bF8RcwEezPuLbud8S6hbKfWvu446Vd3Aia7s2qBXX0P78/SpQ3E7mDAlCJ2DJDuvuITOejp48N+k5imuKNb2vwIQetSLIKijCQdRSpe898QFQhkArY7dzgqQvWnYLmYmaAPauLbqHQPNBPz7ucUpqSnh689NacPnjeZCfojXUMBEX4kFhRW2jGgBz85nxPpqeD16REDxUy5luwT0khDDdzWuri9KqWg6eKGV4uKf22hauISklPx/9mdGBo/F1amwgWosTHDxRSnphZZ9wC1nFLQicvFpNIR3gNYBPZn3CvSPuZWPWRuavv49v3FyQrRgCFShuP/5ujoyP8WNJUkablfoDvQdyyYBLWHxoMQd9wrTPVk35GZpp6xTkaZl1VXqJay9pUwnKEJzUGyo+3rJbyIydA/SfoslNtPJmHeA1gBsTbuTnoz+z8rM5WlCx/1StErVOu+OOD9YaVljWE+xIK2J/VgnzI00ZK14R2vwSr4D0LVrBjRUSQz04dKKUipo6dhwvQkoYFWKvVUdbGIK9+XtJL0tnVlRzte/4YA/0OmHVPfSbqQXntLP6aGWsEJp7qI0UUr1Oz8K4hXw791sG2XvzmK8PN+x8gbSSNKvj9+XvI8A5oJnRVVhnfmIw6YWVVjvnNeW2obfhbu/Ov8r2IZGnr69EBykq1LIKK0Tv6UUAyhBomN0ng+a27BYyM3C2pi+TldTqsOsHXMZAaccT+hKKL3gbhlyuNUsvOKq9VJA7ep1gr4Uh+GzjcVzs9QxzLwGhB3eTKnfCpdrzFlYFQ0zpn3sySth2rBCdgATv+sbXhhYkNugMTIuY1uwcTvZ6Bga4NXIxmfltfw5DQj3wb6GwrU8QEAcn9oGx7e5x4e7hvOsUy8PFVewpPMD8H+bz8vaXqahtnA68L3+fWg10gBlxgTgZ9Hy/o+WgsRkPBw/uGHYH20tT+cXFuccokZYUaams5cZaZQh6HeY4gWURWUvEnKuVtluI0DWjpgLDV1fxRFYmhXYGni3YrAmTgbaMRZOAjvZzZU+mljlUVFHDT7symT80BPuS4+ARqsntArj6a6+780ur7fks3TrbjxcyMNAd17rG8hL1xnqWH13O+JDxuNtbb5+XGK4FjC0L3XJKq0hKK2ometfnCIjTCguLUts1XFd0nIsdgvlh3g/MiJzBO7vfYc73c1h6eClGaaSspozUklQVH+gALg52nBMbwP92Z7WrnesF0RcwyHsQz/l4U2GO2XQzFSVaXLC0vkoZgl6HV5SWfRBuvVl7I1x8NQ2ilgxBXTV8dSWkrmXQnNe4dvB1LD28lDU1JmFWkyEArYexuYjr2+0ZVNcZWTA6QgtcN01LHLoAyrLhyB/NXtKc/rn9eCE7jhcxPMLzZIGcaUWwPWc7OZU5Vt1CZhLDPCmtquNI3kl/68r92rz7bHzATIPURDtdDIWp4BVBgEsA/5rwLz6d/SmBLoE8uPZBrlx2JV8f+hpQ8YGOcv7QEIoqavnzUNs9R/Q6PQ+MfoAcvY538raegdm1TXVZAbVAlbFGxQh6HdMf0XTP23ILmRk4S0tZW/4g/PIA/O9e+PFOWHILfDAbDv8Oc1+BwRdx85Cb6e/Rn0e3/odSV/9GfVbjgz3ILa3mREkVn206xrBwT2KD3Ru+ZBoRM0Pz9+/41OqUhoR58vv+HMqq6xge4XWyAtrUIevnoz/jZOfEpNBJLV6WNe2i3/bnEOLpxFmBvefu5pTwGwSI9hkCoxGK0xoZ6yF+Q/h09qf8a/y/yC7P5vltzwPKEHSU8TG+eLvYt1pTYEmifyLnOYXykb6S40WnIBzYxdRVFPW6XgSgDIGGvQu4dqDyM3ae1t1s8zuw/WPY8422Qji6Wmtic95LMOwq7dR6e54Y9wR5lXk84euN0WJFYJakfnfNEY7klmurgZpyraah6YrAzl5rgHJwGVQ0r1weGuZJjam6eESEN1SYyu6dfak11rLi2Aomh03G2eDc4mX193PF1cGuIWBcWVPP2pRcpg/y7/stFu2dwbtf+wxBaZamWNokY0gndJzX/zx+Ov8nbkq4icsGXoaPk3UJcIV1DHod5yUE8du+E5RWtU8a/K7+F2MvJc+sf+w0z651quvq0VWXUNYLDYFdd0+gV+IVCYva3xBjsN9gbku8jZd3vIxLTQkPS4kQWttKgA/WpeLhZOAvCUFQYMoMspaWmHgFbHoD9nwLo25otMscJ/Bz09xEVORpDXUc3NiYsZai6qIGSYmW0OsEg0M8GgLG61LyqKo1Mq2vxwfMBMS1zxCYVUdbqCp2Njhz29Dbumxatsb8oSF8tOEYv+zJ5uIRYW2O9wsfx81/Psx/dTtYnb6aiaETz8Asm3OiuBp3UUGxKbbXmwyBWhGcIa4ffD3Xew3hGyc7/r3+UaSUuDrY0c/XhTqj5KLhoTga9BYa91HNTxKUoAltJX3WbJc5/XNEhJd2916erwWKheCX1F9ws3djXMi4NueZGO7J/qwSqmrr+f3ACVwd7Bjdr+eojZ5WAuKh4EjbOent7EOgODUSwzyJ8HFut3sIn2gWVNQRqXfhmc3PUF3fvCjyTJBVXIkbFRTbawq4KkagaIYQgjtiLmNhcQlfpHzHf7b+ByklcSb30BWjTc1nGu42WyhUSlygFaadaFwF62Sv51/nx3PrFFN2UsXJYrIt2Vs4O+hs7PVtt5ccEupJbb1kb2Yxv+3PYdIAPxzs+mA1sTUC4gAJOQdaH1d4DBBaZpeiyxFCMD8xhPWH81tUJG2ETo8hII77a505Xnqct3e9ffonaYWs4ircRQVlDpohUCsChVWE7wDuKSjiCt+RfLLvE17a/hI3jI/iifnx9Dc3eyk8plUvWxGLA7T4BGjxiCZcOjK8Ie5ART64+JBdnk1WeRZD/Ye2a45Dwz0B+GTDMXJLq5k2qI8WkVnDrGaZsa31cYWp4B7STN5b0XXMHxqClPDh+tR2pZISlMDY7BTm9p/L+7vf52CB9eLL00lWcRVuVFBu7wQoQ6BoCa9IhNCxyLEfFw+4mPf2vMfa/M+5aozF3b9Z2ril4KxbkBaobtI2sxnleeDsy85crdAm0T+xXVMMcHck0N2RH3ZmohMwZaANGQKPMO0L/njLzYAAU3pv6/LTis4R5evC+Ghf3lh1mOFPruCur5L4ZU92y82TAhOguph/Rl+Ku4M7D69/mDpj85qb00l2cSXe+krK7LXCS2UIFNaxswfPCERBCg+NeYj50fN5Y+cbvLPrnZNjrNUQWCKE1qKvLUNgEpxLyknCUe/IQO/m3Z9aIjHMEylhRKQ3Xi5tu5P6DEJAxFg4tqFVCZH29CFQdJ53F47g3atHMCMukD8O5nDzp9sY+sSv3PTJVtYfbtIgyrSa8yg4yoOjH2Rf/j4+2ffJGZ1vZnEVXrpKyuwMCAQuhp7T27stlCE40/jGQH4KOqHj0bMfZU6/Oby842WWpCzRvnys1RA0O0cbhqCuGqpLwNmHnbk7ifeNx6AztHuKiSb30HRbcguZCT9bK9wrPGp9f22Vlj6qAsWnHUeDnumxATx38RC2Pjidz68fzSUjwth+vIjrPtxKVa3F6sA/VpNhydrFORHnMDVsKq8lvcaxkmNnbL7ZphhBqd4OV4MrOtF7vl57z0z7Cj7RkH8YpESv0/P4uMcZEzSGx9Y/xsajv2oy1219yfgOgLITLfYoMNcZVDq6sz9/f7vdQmamneVPtL8rf0kI7tBxfYKIsdrjsRbcQ8UmgTnlGjqj2Ol1jI325fF58Txz4WAqa+sbi9MZnLTPRfYuhBA8OOZB7HX2PLr+0cZN708jWcWVuMhySnW6XtOr2IwyBGcan/7al32Jprtu0Bl4fvLzRHpEcveG/+Owwa5tt4PvAO3RojitESZ5ib1UUSfrSPRL7NAUYwLc+O3uSYR4OnXouD6B70BNkvpYC72n26ghUJx+Rkf5YNALVic3kaEIOtmbwN/Zn3tH3svWE1v55tA3p31O1XX1FJRV4WisoFT0rvgAKENw5mkiPgfam+b1aa/jgJ5bAvzJc/Zo/Rx+Jn9/C7LUZnmJpGqtujjBL6FTU7YpdDrNPdRSwNhsCFSMoNtwcbBjeIQXaw41iRMEJmhuvVJNNv386PMZHTia57c9T3Z59mmdU05JNa5o6rNlGHtVDQEoQ3Dm8YnRHpvczQe5BvGq/yQK9TpuS3qhmaRxIzwjtKrhluIEJnmJneVpRLpH4uXo1RUztx0ixmqFZaVWvjyKjoHeQetdoeg2JsT4sS+rhLwyi+KxsFHaY9pGQKtHeGTsI9Qb63li4xNtNrzpDJlFlbgL7TNbKutaVPjtqXTKEAghvIUQK4QQyaZHq984QoiFpjHJQoiFVvYvFUK03B6qL+EWBAZnLU7QhLiKMp4trWd/4UEWrVlEvbGFVDm9HXj3b9UQSCCpKLnD8QEFEN5KnMAczNepe6juZEKMpqq7LsViVRCUqHUbTF3XsCnMLYzbh97O6vTVzfuIdyEbjuTjgdkQ1NpcjGAR8LuUMgb43fS8EUIIb+ARYDQwCnjE0mAIIS4Ayjo5j96DTqd9iVuokDZQmMpk5zD+OfKf/JH2B89tfa7l8/jGtGwIyvM4ZmegqKakw/EBBZqv2eAMx63ECQqPKbdQDyAu2AMvZwOrLd1DdvbaquDYukZjFwxawCDvQfxny38or+36lpa19Ua+2Hyc8WFaqnVpfbXNxQjmAR+Zfv8ImG9lzAxghZSyQEpZCKwAZgIIIVyBu4EnOzmP3oVvtPVAr6lQacGgBVw56Eo+3f8p3yd/b/0cfgO1bmd1Nc33VeSR5K7pA6kVwSmgN0DoSOsB48I26jwUZwS9TjAu2pc1ybmNXT6R4zXhQAuFXr1Oz0NjHiKnMoc3kt7o8rn8vv8EJ0qqmR3jhATK6iptLkYQIKXMMv2eDVhznIYAlk1d003bAJ4A/gu04hDXEELcKITYKoTYmpvbdtOKHo1PtPaFYvklXl8LxekNXzL3jriX0UGjeWrTU+zP39/8HL4DQNZrvuymlOeR5OSMm70bUR5WxOsUbRMxTmtmb5miW1kI1cUqdbSHMCHGl5zSapJzLBwKEWMBCcc3Nhqb4JfAhTEX8un+T0kutLIa7wSfbDxGiKcTg30FFUJgRPa9GIEQ4jchxB4rP/Msx0nNLLc7GiOESAT6SylbuOVtjJTybSnlCCnlCD+/DvQO6In4RGtf4uYMFNCMgDQ2uB30Oj3PTHgGDwcP7lp1F8XVxY3PYU4hzbOSOVRRQJJBMMRvSK8qaulRRJwNSEjbfHJboak4SbmGegTjY7TvgdWW3cxCRmjB/CbuIYA7h92Jq70rT216qssCx4dzy1iXks8Vo8PRVZc0NKXpczECKeV0KWW8lZ8fgBNCiCAA02OOlVNkAJai4qGmbWcDI4QQqcBaYIAQYlXnLqeXYCWF1Fp+uo+TD89Pfp4TFSd4YO0DjQtjfE3ZR1biBMUVuRwWHa8fUFgQMgJ0hsZfKKqGoEcR4ulEfz8X1iRbxAkMjhA6wqoh8HL04s5hd7LtxDZ+OvJTl8zhs43HMegFl4wIg6riXtmdDDrvGloKmLOAFgI/WBmzHDhXCOFlChKfCyyXUr4hpQyWUkYC44FDUsrJnZxP78Cnv/ZoaQgaNO4b320O8RvCP0f+k9XpqxtrEtm7aCJpuc0Nwa5areJSxQc6gb0zBCc2Dhi38D9SdB8TYvzYdDS/sdxExFjI2glVJc3GXxhzIYN9B/Pfrf+ltKa0U69dUVPH19vSmBUfhJ+bA1SVnJSgNtiWIXgaOEcIkQxMNz1HCDFCCPEugJSyAC0WsMX087hpm+3i5KU1jWm6ItDZaeqXTbhs4GXMjprNa0mvsT7TIqXRWuaQ0UiSqEGPYLDv4NMzf1sh/GzI2A61ldrzwmPg6Kmpvyp6BBNifKmqNbLdUm4iYpzmZrV065nQCR0Pjn6QgqoCXk96vVOv/ePOTEqr6rjSrB5cVUxpL+xFAJ00BFLKfCnlNClljMmFVGDavlVKeb3FuPellNGmnw+snCdVShnfmbn0OnyaZA4VHtPu8HXNm8AIIXjk7Efo79mf+1bfR1aZKT7vOwDykrVm6maqitjpYM8AB99W+xMr2kHEODDWnuxPUJiq3EI9jDH9zHITFu6hsFHaTdWxtVaPifON45KBl/D5gc9PuW+BlJJPNh5jYIAbIyNN2fBVRZTaa5+5PhcjUJwmmqaQtvEl42xw5oXJL1BnrOPuVXdTU1+jGYLacijNbBhXV3qCXQ72JLq1fC5FOwkfDYiTaaSqD0GPw8XBjmHhXqyx1B2yd4HgoY0Ky5py+9Db8bD34KlNT52SKN3O9GL2ZJRw5dkRWmtYgOoSSg29rxcBKEPQffhEawqiZj9mO75kIj0ieXLck+zJ38O/N//bInPopHsoOXcnlTodid5nna6Z2w5OXpq88bF12qqr6LhaEfRAJsT4sjezidxExDjI3A411jPTPRw8uGv4XezI2cHSw0s7/JqfbDiGi72e84dauHKriikzaEVlyhAo2odl5lB1qaYP1I4vmWkR07h+8PV8c+gbvi415UNbBIyT8jT1xcSA4V09Y9sk4mxI36LJT9fXqNTRHsgEUxppI7mJiHFgrIP05nECM/Oi55Hol8jzW5+nqKqo3a9XWF7Dj7syOX9YCK4Odid3VJVQorfDXmePg753tTFVhqC7aDAEhzucn35b4m2MCx7Hv3a+QpKbd6MVQVJRMv51dQR1oCOZohUixkJNGRwwpRsq11CPIz7EA09nQ+M00vAxIHStuod0QsdDYx6ipKaEF7e/2O7X+3pbGjV1xpNBYjNVxZTp9b0uPgDKEHQf3v0Aoa0IOpifrtfpeWbiMwQ6B3K3jzu5efsa9iWVpzGkugbh4tvlU7ZJzAJ0SV9oj16qUrunYVVuwtFdk6VuqcGQiYHeA7ly0JV8m/wtSTlJbb6W0Sj5bNNxRkV6c1agRfWwlFrWkKDXVRWDMgTdh50DeIZrhqAhPz2y3Yd7OHjw4pQXKRNwT10atfW1nCg/QWZdOYm1UiusUXQe9yDt/3JiNyDAI7S7Z6SwwoRoX06UNJWbGKe59WqrWj32lsRbCHAO4ImNT7TZ8H5NSh7H8itYMCa88Y7aCpD1lEKv0xkCZQi6F59oTYW0MBUc3LXgZAcY6D2QxwMmscOg45kNj7MzdycAibre0zS7V2BeFbiHaAZc0eMYb5KlbuQeihwH9dVa0LgVnA3O3D/qfg4VHuKz/Z+1OvannZm4O9oxMz6w8Y4qTQKmlPpeFygGZQi6F98YU4wgVYsPmNPQOsDMqFn8taiErw4v4dWkV7FHMMhRuYW6lIiztUcVH+ixhHo508/PpXEaabjp/9ZKnMDM1PCpTAqdxGtJr7XYzUxKyfrD+Yzt74uDXZN6H5MhKOuFvQhAGYLuxSdaC0Smbzn1LxnfAdxZWMTZrpEcLT5KfL0Og3MvF+XraUSM0x5VxlCPZmKMHxuPWMhNOHuDf1yLhWWWCCG4f/T9SCl5ZvMzVsccy68go6iScdE+zXea0sBL62tUjEDRQcyaQ5WFp56f7hmBXm/Ps66xxHjFML2qFpytvFEVp453Pxj4Fxg4s7tnomiFc+MCqKo18tUWC9X7yHGa1ER9rfWDDv8BSZ/DvqWEZB/gpojZ/Hb8N1bvX9zsmHWHNbfT2GgrK27ziqC+qlfGCOzaHqI4bZhTSOHUDYHeDnyi8cw/xneXfwtPBYK1OxbFqSMEXP55d89C0QZn9/NhdJQ3r6xM4eIRoTjb22mruc1vQ2YShI08ObiuGn6+D7Y1VrxZCPwUEsi/1j3CyB3f4nTFVw371qfkE+juSD9fKzG4knRqgUpjjYoRKDqIeyjYmbJ7OuN2MIvP1ZRDXZUmaKdQ2BhCCP45cyB5ZdV8sC5V2xhh7j9t4R4qOg7vz9CMwLi/wx074OZ1cO2vGK78jgcH30KGwY63i3Y1HGI0SjYcyWdstM9JSQlL8pIblEdVjEDRMcz9i6Fz0gW+A6DwKJSaxOhUDYHCRhke4c20s/x588/DFFXUgKu/9vkw1xOk/AZvTdSSNC79DM55THP9BcZr2lLR0xg55k7musXwobOelGwt4+hAdikF5TWM69/CZyvvEKXekYCqI1CcCuY4gWd46+Naw3egSXZ3k/ZcxQgUNsy9MwZSVl3Hm3+a2rhGjNVaV656Gj69CNyC4MZVMGhOy+cYsABXo5HHNz6BURpZb4oPjLMWHwDNEJg+w70xRqAMQXcTOw/iL+xcAZi5W5m5iYpyDSlsmEFB7swdEsyH64+SU1IFEeOhugRW/RsSLoHrfzt5A9YCXkGJ3FtQxI7iFL5N/pZ1KXn083Mh0MPK57S2EorSKHXXagtUjEDRcQZfBBe937lzmA2BWS7ZRa0IFLbN3ecMoK5e8vLKZOg/FUJHwuzn4Py3NJnqtvCKZG5lLaPsfXlh2wtsPp7K2P4tfK7yDwOSMldtvzIEiu7B3Lay4LD2XK0IFDZOhI8Ll44M48vNaRyrctRWAaNuaH/Rpk6P8Inm/4weVNZWUe+1tNX4AECpkyegDIGiOzH3JtAZwKH3vREViq7mjmkx2OkFL6xo3te7XfgOIDL/GAmuF2Lw2InOpYVuZnnJgOi13clAGYK+g9kQuPieklSFQtHXCHB3ZOHYSH7YmcmB7OaN7NvEdwAUplKTMxq7+gBe2PE0lXWVzcflHQLPMEqNVQiEChYruhE/kyFQbiGFooG/TeqPq4Mdzy0/hd7Eflo2XnHaYSZ7/42Msgze3Plm83F5h8B3AGU1ZbgYXNCJ3ve12vtmrLBOw4pABYoVCjOezvbcNLEfv+3P4bvt6R072JSEESnTmR87kQtiLuDjvR9zqNDC1WQ0alLyvgMoqSnplfEBUIag7+Br6kimaggUikZcOz6KERFe3L14J/d/t+ukKF1b+GiGYIA+i5GRXtw9/G7cHdx5bMNj1BtN5yjN1HoR+ERTVlPWK+MDoAxB38HFVytKM715FQqFhrO9HV/cOIabJ/Xni81pzH9tHSmWDWxawt6ZEzp/Rrjk4mxvh4eDB/8Y+Q925e7im0PfaGPMbWJ9B1BaW4qbQa0IFN2JEHDzWphwT3fPRKHocRj0OhbNOosP/jqSnNJq5r66tk1XUVFFDfvrghigz2rY9peovzAmaAwvbX+J3IpcU8YQDTECm3QNCSG8hRArhBDJpkerLbaEEAtNY5KFEAstttsLId4WQhwSQhwQQlzYmfnYPI4eYGff3bNQKHosUwb6s+yOCcQHe3D34p384+udVNRYb0+58Ug+h43B+FQd02IBaMJ2D415iOr6ap7d8qy2InDwAFd/m44RLAJ+l1LGAL+bnjdCCOENPAKMBkYBj1gYjAeBHCnlACAW+LOT81EoFIpWCfRw5PMbRnPblGi+2Z7O3FfXsS+zeXrpupR8jutC0ddXQcnJ1UOEewQ3JNzAL6m/sDYvSQsqC0FZbVmvTB2FzhuCecBHpt8/AuZbGTMDWCGlLJBSFgIrAHOHj2uBfwNIKY1SyjwrxysUCkWXYqfXce+MgXxy7WiKK2uZ/9o6Plh3FCllw5h1h/NwCDpLe5LXuCjt2vhrifKI4sn6E1T69EdKabuuISBASml2oGUDAVbGhAAWLYNIB0KEEJ6m508IIbYLIb4WQlg7HgAhxI1CiK1CiK25ubktDVMoFIp2Mz7Gl1/unMD4GF8e+3Ef1320lfyyarKLqziSW05YTKI2MLexIbDX2/N/w+8lQy94S19GZV0l9bJ3Nq6HdhgCIcRvQog9Vn7mWY6TmimVLZzGGnZAKLBeSjkM2AA819JgKeXbUsoRUsoRfn6qJ69CoegafFwdeG/hCB49L5a1KXnMemmNJlYHDBsUDU5ezVYEACN1rswvLeOj4r1sz9H6FvRWQ9Bmq0op5fSW9gkhTgghgqSUWUKIICDHyrAMYLLF81BgFZAPVADfmbZ/DVzXvmkrFApF1yGE4JpxUYzu58PtX+zg803H8XI2MCjIQ6vRsWIIyEvmnoIi/vQJ5rENjwG9U2cIOu8aWorW5hPT4w9WxiwHzhVCeJmCxOcCy00riB85aSSmAfs6OR+FQqE4ZQYFufPjbeO5cWI/7pgWg04nTraCbUreITyl4J7hd5Ndng2Au6H3dSeDzhuCp4FzhBDJwHTTc4QQI4QQ7wJIKQuAJ4Atpp/HTdsA7gMeFULsAq4CVBK8QqHoVpzs9TwwexB/HRelbfAbCOW5UFHQeGDeIfCOYm7MBYwMHAn03hVBm66h1pBS5qPdyTfdvhW43uL5+0Cz7itSymPAxM7MQaFQKE4rZh2vvGStr7EZk8aQEILHxj7G27veJsard1b2q8pihUKhaI0GQ2ChYGqsNxkC7Ys/zC2MJ8Y9gZOdUzdMsPMoQ6BQKBSt4RkOeofGcYKiY1Bf02e0vZQhUCgUitbQ6bU7f8taAguNob6AMgQKhULRFk0zhxpUR9WKQKFQKGwD34GaO6i2Snued0jrBujs3b3z6iKUIVAoFIq28I0BaYSCw9rzvOQ+4xYCZQgUCoWibfxMHQBzTZlDecl9xi0EyhAoFApF2/hEA0IzABUFUJGnVgQKhUJhUxictDTSvIMWGUNqRaBQKBS2he8ALUjcxzKGQBkChUKhaB9+AyEvBXIPgN4ePCO6e0ZdhjIECoVC0R58Y6CuEg7/ocUMdPrunlGXoQyBQqFQtAdfU+ZQzt4+5RYCZQgUCoWifVhmCfWhjCFQhkChUCjah4sPOPtovytDoFAoFDaK2QD4RHfvPLoYZQgUCoWivZgNQR+LEXSqQ5lCoVDYFCOuBZ/+4ODW3TPpUpQhUCgUivYSnKj99DGUa0ihUChsHGUIFAqFwsZRhkChUChsHGUIFAqFwsZRhkChUChsnE4ZAiGEtxBihRAi2fTo1cK4haYxyUKIhRbbLxdC7BZC7BJC/CKE8O3MfBQKhULRcTq7IlgE/C6ljAF+Nz1vhBDCG3gEGA2MAh4RQngJIeyAl4ApUsoEYBdwWyfno1AoFIoO0llDMA/4yPT7R8B8K2NmACuklAVSykJgBTATEKYfFyGEANyBzE7OR6FQKBQdpLMFZQFSyizT79lAgJUxIUCaxfN0IERKWSuE+BuwGygHkoFbW3ohIcSNwI2mp2VCiIOnOGdfIO8Uj+3NqOu2LdR12xbtvW6r3XTaNARCiN+AQCu7HrR8IqWUQgjZjomYz2sA/gYMBY4ArwD3A09aGy+lfBt4u73nb+V1t0opR3T2PL0Ndd22hbpu26Kz192mIZBSTm/lxU8IIYKklFlCiCAgx8qwDGCyxfNQYBWQaDr/YdO5FmMlxqBQKBSK00tnYwRLAXMW0ELgBytjlgPnmgLEXsC5pm0ZQKwQws807hxgfyfno1AoFIoO0tkYwdPAYiHEdcAx4BIAIcQI4GYp5fVSygIhxBPAFtMxj0spC0zjHgNWCyFqTcdf08n5tIdOu5d6Keq6bQt13bZFp65bSNlut75CoVAo+iCqslihUChsHGUIFAqFwsaxGUMghJgphDgohEgRQvTp7CQhxPtCiBwhxB6Lbe2SA+nNCCHChBB/CCH2CSH2CiHuNG3v09cuhHAUQmwWQuw0Xfdjpu1RQohNpvf8V0II++6e6+lACKEXQuwQQvxket7nr1sIkWqS50kSQmw1bTvl97lNGAIhhB54DZgFxAKXCyFiu3dWp5UP0aq3LWlTDqQPUAfcI6WMBcYAt5r+z3392quBqVLKIWhp2TOFEGOAZ4AXpJTRQCFwXfdN8bRyJ40zDm3luqdIKRMt6gdO+X1uE4YATeMoRUp5REpZA3yJJo/RJ5FSrgYKmmxujxxIr0ZKmSWl3G76vRTtyyGEPn7tUqPM9NRg+pHAVOAb0/Y+d90AQohQ4C/Au6bnAhu47hY45fe5rRgCqzIX3TSX7qI9ciB9BiFEJFrV+iZs4NpN7pEktKLOFcBhoEhKWWca0lff8y8C/wSMpuc+2MZ1S+BXIcQ2k/wOdOJ9rprX2yAdlQPpbQghXIFvgb9LKUu0m0SNvnrtUsp6IFEI4Ql8D5zVvTM6/Qgh5gA5UsptQojJ3TydM814KWWGEMIfWCGEOGC5s6Pvc1tZEWQAYRbPQ03bbIkTJhkQWpED6fWYNKy+BT6TUn5n2mwT1w4gpSwC/gDOBjxNcu/QN9/z44C5QohUNHfvVDRp+75+3UgpM0yPOWiGfxSdeJ/biiHYAsSYsgnsgcvQ5DFsifbIgfRqTP7h94D9UsrnLXb16WsXQviZVgIIIZw4KdfyB3CRaVifu24p5f1SylApZSTaZ3qllHIBffy6hRAuQgg38+9osj176MT73GYqi4UQs9H8iXrgfSnlU907o9OHEOILNKE/X+AEWmOgJcBiIByTHIhZ6qOvIIQYD6xBkzY3+4wfQIsT9NlrF0IkoAUH9Wg3d4ullI8LIfqh3Sl7AzuAK6WU1d0309OHyTV0r5RyTl+/btP1fW96agd8LqV8Sgjhwym+z23GECgUCoXCOrbiGlIoFApFCyhDoFAoFDaOMgQKhUJh4yhDoFAoFDaOMgQKhULRQxFCXGwSEjSaGn6155hwIcSvQoj9JgHGyLaOUYZA0aMRQtSbFBb3CCG+FkI494A5TRZCjO3kOYKFEN+0PbLRMdcIIV41/X6zEOLqzsxB0bMwva8+bLJ5D3ABsLoDp/oY+I+UchBaoVmbhWXKECh6OpUmhcV4oAa4uT0HWVSWng4mAx0yBE3nI6XMlFJe1NL4tpBSviml/PhUj1f0DqSU+6WUB5tuN2lL/UcIsUUIsUsIcZNpeyxgJ6VcYTq+TEpZ0dbrKEOg6E2sAaKFEOeZ9OZ3CCF+E0IEAAghHhVCfCKEWAd8IoSIFEKsEUJsN/2MNY2bLIT4UwjxgxDiiBDiaSHEAqFp+u8WQvQ3jfMTQnxr+rBtEUKMMy2zbwbuMq1UJlgbZ20+lhdimtse0+/XCCG+E0L8IjQt+Wctxv1VCHFICLEZTVIBi3Pfa/o92vR32Gm6TvP8/2HxRWHuUeAihPifaeweIcSlp+U/pTjdXAcUSylHAiOBG4QQUcAAoMj0ftphMhb6tk6mROcUvQLTHfUs4BdgLTDGJKx1PZr65D2mobFoglyVJjfSOVLKKiFEDPAFYPazDgEGocl1HwHelVKOElozm9uBv6Pp1rwgpVwrhAgHlkspBwkh3gTKpJTPmeb2edNxpnM3mk8bl5iIppZaDRwUQryC1l/hMWA4UIwmnbDDyrGfAU9LKb8XQjgCOiHEuUAMmmtAAEuFEBMBPyBTSvkX09w92piX4jQjhNgEOACugLfQVGQB7pNSLm/hsHOBBCGEeVXpgfb/tgMmoL2XjgNfAdegSa+0iDIEip6Ok8UHYw3aG3og8JXQhLXsgaMW45dafOkagFeFEIlAPdrdkpktZsleIcRh4FfT9t3AFNPv04FYcVK91F1oyqZNaW3c0nYYAdAaihSb5rMPiECTCFklpcw1bf+qyTUgNM2ZECnl9wBSyirT9nPRvizMhsMV7YtiDfBfIcQzwE9SyjXtmJviNCKlHA0NMhnXSCmvacdhAri9qaEQWkOiJCnlEdPzJWhNmpQhUPRqKqWUiZYbTHfLz0spl5o+PI9a7C63+P0uNK2lIWhu0CqLfZbaM0aL50ZOfi50aCsPy+Ow+MKnHePKmw5uAcv51NP5z6YA/i2lfKvZDiGGAbOBJ4UQv0spH+/kaynOPMuBvwkhVkopa4UQA9BUVregqa/6mW4gpgJb2zqZihEoeiMenJQWXtjGuCwppRG4Ck2UrSP8iuYmAsC0sgAoBdzaMa6zbAImCSF8hCavfXHTAaZObOlCiPmm13YwucSWA9eaVyZCiBAhhL8QIhiokFJ+CvwHGNZFc1WcBoQQ5wsh0tFkxf8nhDCvAN4F9gHbTbGmt9CCxPXAvcDvQojdaDcE77T1OmpFoOiNPAp8LYQoBFYCUS2Mex34Vmhplr/Q/rtzM3cArwkhdqF9VlajBYp/BL4RQsxDMwAtjesUUsosIcSjwAagCEhqYehVwFtCiMeBWuBiKeWvQohBwAbTyqQMuBKIBv4jhDCaxv6ts/NUdA1SylXAqibbvuek0qjldiOasu4DVvatABI68tpKfVShUChsHOUaUigUChtHGQKFQqGwcZQhUCgUChtHGQKFQqGwcZQhUCgUChtHGQKFQqGwcZQhUCgUChvn/wHqt6bqXWGpygAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "plt.plot(np.arange(1000000,1000050,1), reverse_top10fft[1000000:1000050], label = \"FFT top-10%\")\n",
+ "plt.plot(np.arange(1000000,1000050,1), concr[1000000:1000050], label = \"Original Parameters\")\n",
+ "plt.plot(np.arange(1000000,1000050,1), reverse_top10wv[1000000:1000050], label = \"Sym2 top-10%\")\n",
+ "\n",
+ "plt.title('Parameter Values') \n",
+ "#plt.ylabel(\"Absolute frequency value\")\n",
+ "plt.xlabel(\"Parameter indices\")\n",
+ "plt.legend()\n",
+ "plt.draw()\n",
+ "plt.savefig(\"ParametersWaveletHaar.png\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## FFT Training<a class=\"anchor\" id=\"ffttrain\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 85,
+ "metadata": {
+ "id": "e65Izyv0s-yE"
+ },
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "eqOXilqMs-yF",
+ "outputId": "06799a3b-983b-4f51-a7bd-a901c041bd05"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.136391 [ 0/735856]\n",
+ "loss: 1.387546 [64000/735856]\n",
+ "loss: 1.009362 [128000/735856]\n",
+ "loss: 0.568759 [192000/735856]\n",
+ "loss: 0.796950 [256000/735856]\n",
+ "loss: 0.670068 [320000/735856]\n",
+ "loss: 0.625332 [384000/735856]\n",
+ "loss: 0.557147 [448000/735856]\n",
+ "loss: 0.701893 [512000/735856]\n",
+ "loss: 0.670033 [576000/735856]\n",
+ "loss: 0.575888 [640000/735856]\n",
+ "loss: 0.654841 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 82.0%, Avg loss: 0.578153 \n",
+ "\n",
+ "loss: 0.776733 [ 0/735856]\n",
+ "loss: 0.519993 [64000/735856]\n",
+ "loss: 0.599282 [128000/735856]\n",
+ "loss: 0.885723 [192000/735856]\n",
+ "loss: 0.514714 [256000/735856]\n",
+ "loss: 0.539040 [320000/735856]\n",
+ "loss: 0.422559 [384000/735856]\n",
+ "loss: 0.382564 [448000/735856]\n",
+ "loss: 0.412677 [512000/735856]\n",
+ "loss: 0.360731 [576000/735856]\n",
+ "loss: 0.534333 [640000/735856]\n",
+ "loss: 0.379236 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.1%, Avg loss: 0.463718 \n",
+ "\n",
+ "loss: 0.367664 [ 0/735856]\n",
+ "loss: 0.339760 [64000/735856]\n",
+ "loss: 0.653718 [128000/735856]\n",
+ "loss: 0.410070 [192000/735856]\n",
+ "loss: 0.554535 [256000/735856]\n",
+ "loss: 0.578007 [320000/735856]\n",
+ "loss: 0.421670 [384000/735856]\n",
+ "loss: 0.599983 [448000/735856]\n",
+ "loss: 0.262858 [512000/735856]\n",
+ "loss: 0.333737 [576000/735856]\n",
+ "loss: 0.361296 [640000/735856]\n",
+ "loss: 0.468058 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.9%, Avg loss: 0.420345 \n",
+ "\n",
+ "loss: 0.443198 [ 0/735856]\n",
+ "loss: 0.507409 [64000/735856]\n",
+ "loss: 0.554008 [128000/735856]\n",
+ "loss: 0.304086 [192000/735856]\n",
+ "loss: 0.482780 [256000/735856]\n",
+ "loss: 0.349616 [320000/735856]\n",
+ "loss: 0.402055 [384000/735856]\n",
+ "loss: 0.345523 [448000/735856]\n",
+ "loss: 0.364194 [512000/735856]\n",
+ "loss: 0.310542 [576000/735856]\n",
+ "loss: 0.441185 [640000/735856]\n",
+ "loss: 0.276955 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.8%, Avg loss: 0.390410 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " topk = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat))\n",
+ " top10[topk.indices] = flat[topk.indices]\n",
+ " g.grad = top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# it converges slower than without gradient compression"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch.fft as fft"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.122382 [ 0/735856]\n",
+ "loss: 1.659356 [64000/735856]\n",
+ "loss: 1.175072 [128000/735856]\n",
+ "loss: 1.030752 [192000/735856]\n",
+ "loss: 0.891644 [256000/735856]\n",
+ "loss: 0.732518 [320000/735856]\n",
+ "loss: 0.613185 [384000/735856]\n",
+ "loss: 0.483264 [448000/735856]\n",
+ "loss: 0.580724 [512000/735856]\n",
+ "loss: 0.509457 [576000/735856]\n",
+ "loss: 0.661517 [640000/735856]\n",
+ "loss: 0.621521 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 81.7%, Avg loss: 0.570322 \n",
+ "\n",
+ "loss: 0.543810 [ 0/735856]\n",
+ "loss: 0.339085 [64000/735856]\n",
+ "loss: 0.495473 [128000/735856]\n",
+ "loss: 0.384833 [192000/735856]\n",
+ "loss: 0.418521 [256000/735856]\n",
+ "loss: 0.614597 [320000/735856]\n",
+ "loss: 0.515266 [384000/735856]\n",
+ "loss: 0.738823 [448000/735856]\n",
+ "loss: 0.423178 [512000/735856]\n",
+ "loss: 0.473593 [576000/735856]\n",
+ "loss: 0.518021 [640000/735856]\n",
+ "loss: 0.497685 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 84.0%, Avg loss: 0.474809 \n",
+ "\n",
+ "loss: 0.575689 [ 0/735856]\n",
+ "loss: 0.456497 [64000/735856]\n",
+ "loss: 0.429356 [128000/735856]\n",
+ "loss: 0.563055 [192000/735856]\n",
+ "loss: 0.486054 [256000/735856]\n",
+ "loss: 0.542747 [320000/735856]\n",
+ "loss: 0.441926 [384000/735856]\n",
+ "loss: 0.461542 [448000/735856]\n",
+ "loss: 0.502812 [512000/735856]\n",
+ "loss: 0.383888 [576000/735856]\n",
+ "loss: 0.266721 [640000/735856]\n",
+ "loss: 0.490470 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.8%, Avg loss: 0.423047 \n",
+ "\n",
+ "loss: 0.302303 [ 0/735856]\n",
+ "loss: 0.421864 [64000/735856]\n",
+ "loss: 0.376742 [128000/735856]\n",
+ "loss: 0.259237 [192000/735856]\n",
+ "loss: 0.368860 [256000/735856]\n",
+ "loss: 0.400204 [320000/735856]\n",
+ "loss: 0.310619 [384000/735856]\n",
+ "loss: 0.320007 [448000/735856]\n",
+ "loss: 0.305337 [512000/735856]\n",
+ "loss: 0.375540 [576000/735856]\n",
+ "loss: 0.362421 [640000/735856]\n",
+ "loss: 0.347816 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.3%, Avg loss: 0.400034 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " \n",
+ " flat_fft = fft.rfft(flat)\n",
+ " topk = torch.topk(flat_fft.abs(), round(0.1*len(flat_fft)), dim=0, sorted=False)\n",
+ " top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ " top10[topk.indices] = flat_fft[topk.indices]\n",
+ " reverse_top10 = fft.irfft(top10)\n",
+ " g.grad = reverse_top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.125680 [ 0/735856]\n",
+ "loss: 1.490153 [64000/735856]\n",
+ "loss: 0.797238 [128000/735856]\n",
+ "loss: 0.703639 [192000/735856]\n",
+ "loss: 0.862654 [256000/735856]\n",
+ "loss: 0.674491 [320000/735856]\n",
+ "loss: 0.633835 [384000/735856]\n",
+ "loss: 0.537149 [448000/735856]\n",
+ "loss: 0.579062 [512000/735856]\n",
+ "loss: 0.468447 [576000/735856]\n",
+ "loss: 0.488582 [640000/735856]\n",
+ "loss: 0.529873 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 83.8%, Avg loss: 0.489548 \n",
+ "\n",
+ "loss: 0.573154 [ 0/735856]\n",
+ "loss: 0.466781 [64000/735856]\n",
+ "loss: 0.468422 [128000/735856]\n",
+ "loss: 0.449423 [192000/735856]\n",
+ "loss: 0.357713 [256000/735856]\n",
+ "loss: 0.391187 [320000/735856]\n",
+ "loss: 0.500866 [384000/735856]\n",
+ "loss: 0.368405 [448000/735856]\n",
+ "loss: 0.423239 [512000/735856]\n",
+ "loss: 0.533780 [576000/735856]\n",
+ "loss: 0.623185 [640000/735856]\n",
+ "loss: 0.380635 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.9%, Avg loss: 0.411804 \n",
+ "\n",
+ "loss: 0.485906 [ 0/735856]\n",
+ "loss: 0.522850 [64000/735856]\n",
+ "loss: 0.474864 [128000/735856]\n",
+ "loss: 0.453226 [192000/735856]\n",
+ "loss: 0.311791 [256000/735856]\n",
+ "loss: 0.370382 [320000/735856]\n",
+ "loss: 0.415271 [384000/735856]\n",
+ "loss: 0.448348 [448000/735856]\n",
+ "loss: 0.416761 [512000/735856]\n",
+ "loss: 0.392923 [576000/735856]\n",
+ "loss: 0.408733 [640000/735856]\n",
+ "loss: 0.369844 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.8%, Avg loss: 0.382454 \n",
+ "\n",
+ "loss: 0.351067 [ 0/735856]\n",
+ "loss: 0.441320 [64000/735856]\n",
+ "loss: 0.376012 [128000/735856]\n",
+ "loss: 0.326137 [192000/735856]\n",
+ "loss: 0.326353 [256000/735856]\n",
+ "loss: 0.337223 [320000/735856]\n",
+ "loss: 0.377199 [384000/735856]\n",
+ "loss: 0.453688 [448000/735856]\n",
+ "loss: 0.394669 [512000/735856]\n",
+ "loss: 0.462621 [576000/735856]\n",
+ "loss: 0.365274 [640000/735856]\n",
+ "loss: 0.414022 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.0%, Avg loss: 0.381759 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " \n",
+ " flat_fft = fft.rfft(flat)\n",
+ " topk = torch.topk(flat_fft.abs(), round(0.2*len(flat_fft)), dim=0, sorted=False)\n",
+ " top10 = torch.zeros(len(flat_fft), dtype = torch.cfloat)\n",
+ " top10[topk.indices] = flat_fft[topk.indices]\n",
+ " reverse_top10 = fft.irfft(top10)\n",
+ " g.grad = reverse_top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 229,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 230,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.128546 [ 0/735856]\n",
+ "loss: 1.380654 [64000/735856]\n",
+ "loss: 1.055664 [128000/735856]\n",
+ "loss: 0.687121 [192000/735856]\n",
+ "loss: 0.728443 [256000/735856]\n",
+ "loss: 0.731651 [320000/735856]\n",
+ "loss: 0.649674 [384000/735856]\n",
+ "loss: 0.474646 [448000/735856]\n",
+ "loss: 0.653415 [512000/735856]\n",
+ "loss: 0.450781 [576000/735856]\n",
+ "loss: 0.629819 [640000/735856]\n",
+ "loss: 0.548388 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 83.2%, Avg loss: 0.540368 \n",
+ "\n",
+ "loss: 0.767534 [ 0/735856]\n",
+ "loss: 0.474996 [64000/735856]\n",
+ "loss: 0.657538 [128000/735856]\n",
+ "loss: 0.388315 [192000/735856]\n",
+ "loss: 0.581206 [256000/735856]\n",
+ "loss: 0.421425 [320000/735856]\n",
+ "loss: 0.494563 [384000/735856]\n",
+ "loss: 0.541493 [448000/735856]\n",
+ "loss: 0.451657 [512000/735856]\n",
+ "loss: 0.382599 [576000/735856]\n",
+ "loss: 0.449485 [640000/735856]\n",
+ "loss: 0.408576 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.0%, Avg loss: 0.455863 \n",
+ "\n",
+ "loss: 0.487152 [ 0/735856]\n",
+ "loss: 0.566136 [64000/735856]\n",
+ "loss: 0.388435 [128000/735856]\n",
+ "loss: 0.435407 [192000/735856]\n",
+ "loss: 0.626423 [256000/735856]\n",
+ "loss: 0.436673 [320000/735856]\n",
+ "loss: 0.599878 [384000/735856]\n",
+ "loss: 0.567672 [448000/735856]\n",
+ "loss: 0.458641 [512000/735856]\n",
+ "loss: 0.479425 [576000/735856]\n",
+ "loss: 0.289777 [640000/735856]\n",
+ "loss: 0.392798 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.1%, Avg loss: 0.449504 \n",
+ "\n",
+ "loss: 0.616642 [ 0/735856]\n",
+ "loss: 0.266790 [64000/735856]\n",
+ "loss: 0.314584 [128000/735856]\n",
+ "loss: 0.314711 [192000/735856]\n",
+ "loss: 0.429452 [256000/735856]\n",
+ "loss: 0.363823 [320000/735856]\n",
+ "loss: 0.594678 [384000/735856]\n",
+ "loss: 0.417127 [448000/735856]\n",
+ "loss: 0.415177 [512000/735856]\n",
+ "loss: 0.406279 [576000/735856]\n",
+ "loss: 0.512797 [640000/735856]\n",
+ "loss: 0.259631 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.3%, Avg loss: 0.415515 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavelet per layer\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'haar'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " \n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ " g.grad = reverse_top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.114821 [ 0/735856]\n",
+ "loss: 1.691283 [64000/735856]\n",
+ "loss: 0.739705 [128000/735856]\n",
+ "loss: 0.878835 [192000/735856]\n",
+ "loss: 0.893373 [256000/735856]\n",
+ "loss: 0.622142 [320000/735856]\n",
+ "loss: 0.729517 [384000/735856]\n",
+ "loss: 0.930510 [448000/735856]\n",
+ "loss: 0.564309 [512000/735856]\n",
+ "loss: 0.820855 [576000/735856]\n",
+ "loss: 0.592394 [640000/735856]\n",
+ "loss: 0.530982 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 82.1%, Avg loss: 0.576240 \n",
+ "\n",
+ "loss: 0.387244 [ 0/735856]\n",
+ "loss: 0.483110 [64000/735856]\n",
+ "loss: 0.544743 [128000/735856]\n",
+ "loss: 0.570393 [192000/735856]\n",
+ "loss: 0.511510 [256000/735856]\n",
+ "loss: 0.335736 [320000/735856]\n",
+ "loss: 0.671059 [384000/735856]\n",
+ "loss: 0.473634 [448000/735856]\n",
+ "loss: 0.559810 [512000/735856]\n",
+ "loss: 0.454633 [576000/735856]\n",
+ "loss: 0.571824 [640000/735856]\n",
+ "loss: 0.626598 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 84.3%, Avg loss: 0.482487 \n",
+ "\n",
+ "loss: 0.422876 [ 0/735856]\n",
+ "loss: 0.769186 [64000/735856]\n",
+ "loss: 0.351542 [128000/735856]\n",
+ "loss: 0.436626 [192000/735856]\n",
+ "loss: 0.628383 [256000/735856]\n",
+ "loss: 0.528591 [320000/735856]\n",
+ "loss: 0.573713 [384000/735856]\n",
+ "loss: 0.517758 [448000/735856]\n",
+ "loss: 0.434379 [512000/735856]\n",
+ "loss: 0.491439 [576000/735856]\n",
+ "loss: 0.494193 [640000/735856]\n",
+ "loss: 0.505279 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.1%, Avg loss: 0.454134 \n",
+ "\n",
+ "loss: 0.439892 [ 0/735856]\n",
+ "loss: 0.459202 [64000/735856]\n",
+ "loss: 0.245611 [128000/735856]\n",
+ "loss: 0.355409 [192000/735856]\n",
+ "loss: 0.490522 [256000/735856]\n",
+ "loss: 0.481495 [320000/735856]\n",
+ "loss: 0.426439 [384000/735856]\n",
+ "loss: 0.641797 [448000/735856]\n",
+ "loss: 0.423894 [512000/735856]\n",
+ "loss: 0.498421 [576000/735856]\n",
+ "loss: 0.344970 [640000/735856]\n",
+ "loss: 0.368346 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 84.2%, Avg loss: 0.466052 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# per layer repeat\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'sym2'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " \n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " flat = g.grad.flatten()\n",
+ " \n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = 4)\n",
+ " # print(len(coeff))\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ " g.grad = reverse_top10.reshape(shape)\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.135039 [ 0/735856]\n",
+ "loss: 1.154176 [64000/735856]\n",
+ "loss: 0.624926 [128000/735856]\n",
+ "loss: 0.605651 [192000/735856]\n",
+ "loss: 0.601686 [256000/735856]\n",
+ "loss: 0.532184 [320000/735856]\n",
+ "loss: 0.627395 [384000/735856]\n",
+ "loss: 0.411491 [448000/735856]\n",
+ "loss: 0.354714 [512000/735856]\n",
+ "loss: 0.393673 [576000/735856]\n",
+ "loss: 0.612208 [640000/735856]\n",
+ "loss: 0.619142 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.2%, Avg loss: 0.445382 \n",
+ "\n",
+ "loss: 0.429652 [ 0/735856]\n",
+ "loss: 0.396769 [64000/735856]\n",
+ "loss: 0.423508 [128000/735856]\n",
+ "loss: 0.576669 [192000/735856]\n",
+ "loss: 0.432909 [256000/735856]\n",
+ "loss: 0.515018 [320000/735856]\n",
+ "loss: 0.375972 [384000/735856]\n",
+ "loss: 0.376615 [448000/735856]\n",
+ "loss: 0.326449 [512000/735856]\n",
+ "loss: 0.360019 [576000/735856]\n",
+ "loss: 0.354862 [640000/735856]\n",
+ "loss: 0.522963 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.9%, Avg loss: 0.383029 \n",
+ "\n",
+ "loss: 0.319733 [ 0/735856]\n",
+ "loss: 0.486813 [64000/735856]\n",
+ "loss: 0.351780 [128000/735856]\n",
+ "loss: 0.327754 [192000/735856]\n",
+ "loss: 0.311207 [256000/735856]\n",
+ "loss: 0.421759 [320000/735856]\n",
+ "loss: 0.486802 [384000/735856]\n",
+ "loss: 0.327473 [448000/735856]\n",
+ "loss: 0.229189 [512000/735856]\n",
+ "loss: 0.395156 [576000/735856]\n",
+ "loss: 0.330383 [640000/735856]\n",
+ "loss: 0.240293 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.1%, Avg loss: 0.374847 \n",
+ "\n",
+ "loss: 0.364167 [ 0/735856]\n",
+ "loss: 0.325380 [64000/735856]\n",
+ "loss: 0.407133 [128000/735856]\n",
+ "loss: 0.229438 [192000/735856]\n",
+ "loss: 0.324557 [256000/735856]\n",
+ "loss: 0.312494 [320000/735856]\n",
+ "loss: 0.250331 [384000/735856]\n",
+ "loss: 0.405609 [448000/735856]\n",
+ "loss: 0.334161 [512000/735856]\n",
+ "loss: 0.305596 [576000/735856]\n",
+ "loss: 0.396855 [640000/735856]\n",
+ "loss: 0.267720 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.3%, Avg loss: 0.357564 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavelet over entire flatten gradient\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'haar'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " flats = []\n",
+ " shapes = []\n",
+ " lens = []\n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " shapes.append(shape)\n",
+ " flat = g.grad.flatten()\n",
+ " flats.append(flat)\n",
+ " lens.append(len(flat))\n",
+ " flat = torch.cat(flats)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ "\n",
+ " \n",
+ " start_index = 0 \n",
+ " for i, key in enumerate(model.parameters()):\n",
+ " end_index = start_index + lens[i]\n",
+ " key.grad = reverse_top10[start_index:end_index].reshape(shapes[i])\n",
+ " start_index = end_index\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pywt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.126535 [ 0/735856]\n",
+ "loss: 1.238513 [64000/735856]\n",
+ "loss: 0.947478 [128000/735856]\n",
+ "loss: 0.758107 [192000/735856]\n",
+ "loss: 0.538468 [256000/735856]\n",
+ "loss: 0.726651 [320000/735856]\n",
+ "loss: 0.523160 [384000/735856]\n",
+ "loss: 0.323133 [448000/735856]\n",
+ "loss: 0.439029 [512000/735856]\n",
+ "loss: 0.406259 [576000/735856]\n",
+ "loss: 0.490085 [640000/735856]\n",
+ "loss: 0.520512 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.0%, Avg loss: 0.449423 \n",
+ "\n",
+ "loss: 0.481995 [ 0/735856]\n",
+ "loss: 0.485922 [64000/735856]\n",
+ "loss: 0.363491 [128000/735856]\n",
+ "loss: 0.604679 [192000/735856]\n",
+ "loss: 0.318160 [256000/735856]\n",
+ "loss: 0.321950 [320000/735856]\n",
+ "loss: 0.355750 [384000/735856]\n",
+ "loss: 0.399116 [448000/735856]\n",
+ "loss: 0.283532 [512000/735856]\n",
+ "loss: 0.527641 [576000/735856]\n",
+ "loss: 0.413641 [640000/735856]\n",
+ "loss: 0.309524 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.0%, Avg loss: 0.406266 \n",
+ "\n",
+ "loss: 0.332639 [ 0/735856]\n",
+ "loss: 0.438504 [64000/735856]\n",
+ "loss: 0.375174 [128000/735856]\n",
+ "loss: 0.325330 [192000/735856]\n",
+ "loss: 0.311181 [256000/735856]\n",
+ "loss: 0.439757 [320000/735856]\n",
+ "loss: 0.357552 [384000/735856]\n",
+ "loss: 0.318609 [448000/735856]\n",
+ "loss: 0.265860 [512000/735856]\n",
+ "loss: 0.534769 [576000/735856]\n",
+ "loss: 0.287946 [640000/735856]\n",
+ "loss: 0.381077 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.0%, Avg loss: 0.377133 \n",
+ "\n",
+ "loss: 0.293545 [ 0/735856]\n",
+ "loss: 0.346547 [64000/735856]\n",
+ "loss: 0.489387 [128000/735856]\n",
+ "loss: 0.438751 [192000/735856]\n",
+ "loss: 0.376747 [256000/735856]\n",
+ "loss: 0.427431 [320000/735856]\n",
+ "loss: 0.381158 [384000/735856]\n",
+ "loss: 0.482535 [448000/735856]\n",
+ "loss: 0.229551 [512000/735856]\n",
+ "loss: 0.455859 [576000/735856]\n",
+ "loss: 0.332654 [640000/735856]\n",
+ "loss: 0.496725 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.5%, Avg loss: 0.358163 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# rerun with alpha 0.2\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'sym2'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " flats = []\n",
+ " shapes = []\n",
+ " lens = []\n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " shapes.append(shape)\n",
+ " flat = g.grad.flatten()\n",
+ " flats.append(flat)\n",
+ " lens.append(len(flat))\n",
+ " flat = torch.cat(flats)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = None)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ "\n",
+ " \n",
+ " start_index = 0 \n",
+ " for i, key in enumerate(model.parameters()):\n",
+ " end_index = start_index + lens[i]\n",
+ " key.grad = reverse_top10[start_index:end_index].reshape(shapes[i])\n",
+ " start_index = end_index\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "v,i = torch.topk(torch.tensor([1,2,3,4]), 2 , dim=0, sorted=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([4, 3])"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "v"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([3, 2])"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "i"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.126898 [ 0/735856]\n",
+ "loss: 1.050626 [64000/735856]\n",
+ "loss: 0.652647 [128000/735856]\n",
+ "loss: 0.648297 [192000/735856]\n",
+ "loss: 0.636182 [256000/735856]\n",
+ "loss: 0.570731 [320000/735856]\n",
+ "loss: 0.509262 [384000/735856]\n",
+ "loss: 0.309913 [448000/735856]\n",
+ "loss: 0.538662 [512000/735856]\n",
+ "loss: 0.530801 [576000/735856]\n",
+ "loss: 0.507737 [640000/735856]\n",
+ "loss: 0.422813 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 85.5%, Avg loss: 0.435624 \n",
+ "\n",
+ "loss: 0.388328 [ 0/735856]\n",
+ "loss: 0.299637 [64000/735856]\n",
+ "loss: 0.420440 [128000/735856]\n",
+ "loss: 0.230143 [192000/735856]\n",
+ "loss: 0.374027 [256000/735856]\n",
+ "loss: 0.279048 [320000/735856]\n",
+ "loss: 0.495672 [384000/735856]\n",
+ "loss: 0.277394 [448000/735856]\n",
+ "loss: 0.395940 [512000/735856]\n",
+ "loss: 0.476103 [576000/735856]\n",
+ "loss: 0.550471 [640000/735856]\n",
+ "loss: 0.431940 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.7%, Avg loss: 0.391393 \n",
+ "\n",
+ "loss: 0.436391 [ 0/735856]\n",
+ "loss: 0.351771 [64000/735856]\n",
+ "loss: 0.352133 [128000/735856]\n",
+ "loss: 0.254270 [192000/735856]\n",
+ "loss: 0.357840 [256000/735856]\n",
+ "loss: 0.368416 [320000/735856]\n",
+ "loss: 0.401375 [384000/735856]\n",
+ "loss: 0.442322 [448000/735856]\n",
+ "loss: 0.538914 [512000/735856]\n",
+ "loss: 0.444955 [576000/735856]\n",
+ "loss: 0.322195 [640000/735856]\n",
+ "loss: 0.493332 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.9%, Avg loss: 0.374818 \n",
+ "\n",
+ "loss: 0.457855 [ 0/735856]\n",
+ "loss: 0.423867 [64000/735856]\n",
+ "loss: 0.274726 [128000/735856]\n",
+ "loss: 0.356364 [192000/735856]\n",
+ "loss: 0.341427 [256000/735856]\n",
+ "loss: 0.301665 [320000/735856]\n",
+ "loss: 0.409492 [384000/735856]\n",
+ "loss: 0.401218 [448000/735856]\n",
+ "loss: 0.616257 [512000/735856]\n",
+ "loss: 0.287706 [576000/735856]\n",
+ "loss: 0.321826 [640000/735856]\n",
+ "loss: 0.423405 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.6%, Avg loss: 0.357925 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# rerun with alpha 0.2 and level 4\n",
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "wavelet = 'coif1'\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " flats = []\n",
+ " shapes = []\n",
+ " lens = []\n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " shapes.append(shape)\n",
+ " flat = g.grad.flatten()\n",
+ " flats.append(flat)\n",
+ " lens.append(len(flat))\n",
+ " flat = torch.cat(flats)\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = 4)\n",
+ " # print(len(coeff))\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.2*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " \n",
+ "\n",
+ " \n",
+ " start_index = 0 \n",
+ " for i, key in enumerate(model.parameters()):\n",
+ " end_index = start_index + lens[i]\n",
+ " key.grad = reverse_top10[start_index:end_index].reshape(shapes[i])\n",
+ " start_index = end_index\n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.0005\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.118358 [ 0/735856]\n",
+ "loss: 1.330925 [64000/735856]\n",
+ "loss: 0.899926 [128000/735856]\n",
+ "loss: 0.894990 [192000/735856]\n",
+ "loss: 0.475845 [256000/735856]\n",
+ "loss: 0.672299 [320000/735856]\n",
+ "loss: 0.728748 [384000/735856]\n",
+ "loss: 0.374176 [448000/735856]\n",
+ "loss: 0.621309 [512000/735856]\n",
+ "loss: 0.562943 [576000/735856]\n",
+ "loss: 0.567177 [640000/735856]\n",
+ "loss: 0.408742 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 84.7%, Avg loss: 0.461047 \n",
+ "\n",
+ "loss: 0.545014 [ 0/735856]\n",
+ "loss: 0.433877 [64000/735856]\n",
+ "loss: 0.513009 [128000/735856]\n",
+ "loss: 0.462199 [192000/735856]\n",
+ "loss: 0.371584 [256000/735856]\n",
+ "loss: 0.380919 [320000/735856]\n",
+ "loss: 0.448126 [384000/735856]\n",
+ "loss: 0.421078 [448000/735856]\n",
+ "loss: 0.531703 [512000/735856]\n",
+ "loss: 0.314307 [576000/735856]\n",
+ "loss: 0.345081 [640000/735856]\n",
+ "loss: 0.456303 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.6%, Avg loss: 0.392272 \n",
+ "\n",
+ "loss: 0.371980 [ 0/735856]\n",
+ "loss: 0.419902 [64000/735856]\n",
+ "loss: 0.344231 [128000/735856]\n",
+ "loss: 0.383977 [192000/735856]\n",
+ "loss: 0.586718 [256000/735856]\n",
+ "loss: 0.524982 [320000/735856]\n",
+ "loss: 0.333949 [384000/735856]\n",
+ "loss: 0.478536 [448000/735856]\n",
+ "loss: 0.346808 [512000/735856]\n",
+ "loss: 0.322247 [576000/735856]\n",
+ "loss: 0.281340 [640000/735856]\n",
+ "loss: 0.373933 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 86.8%, Avg loss: 0.379021 \n",
+ "\n",
+ "loss: 0.393135 [ 0/735856]\n",
+ "loss: 0.281718 [64000/735856]\n",
+ "loss: 0.488630 [128000/735856]\n",
+ "loss: 0.335369 [192000/735856]\n",
+ "loss: 0.342869 [256000/735856]\n",
+ "loss: 0.293455 [320000/735856]\n",
+ "loss: 0.391644 [384000/735856]\n",
+ "loss: 0.309957 [448000/735856]\n",
+ "loss: 0.277645 [512000/735856]\n",
+ "loss: 0.277113 [576000/735856]\n",
+ "loss: 0.242315 [640000/735856]\n",
+ "loss: 0.292711 [704000/735856]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 87.3%, Avg loss: 0.366945 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(4):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " # print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " \n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " # print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " \n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " # print(\"params \"+ str(len(list(model.parameters()))))\n",
+ " flats = []\n",
+ " shapes = []\n",
+ " lens = []\n",
+ " for g in model.parameters():\n",
+ " grad = g.grad\n",
+ " shape = grad.shape\n",
+ " shapes.append(shape)\n",
+ " flat = g.grad.flatten()\n",
+ " flats.append(flat)\n",
+ " lens.append(len(flat))\n",
+ " flat = torch.cat(flats)\n",
+ "\n",
+ " topk = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat))\n",
+ " top10[topk.indices] = flat[topk.indices]\n",
+ " \n",
+ " start_index = 0 \n",
+ " for i, key in enumerate(model.parameters()):\n",
+ " end_index = start_index + lens[i]\n",
+ " key.grad = top10[start_index:end_index].reshape(shapes[i])\n",
+ " start_index = end_index\n",
+ " \n",
+ " \n",
+ " # print(\"grad3: \"+str(model.conv1.bias.grad))\n",
+ " \n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " # print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Node Training <a class=\"anchor\" id=\"nodetraining\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "id": "9LpgzEw1s-xo"
+ },
+ "outputs": [],
+ "source": [
+ "# From Femnist.py\n",
+ "def read_file(file_path):\n",
+ " with open(file_path, \"r\") as inf:\n",
+ " client_data = json.load(inf)\n",
+ " print(\"loaded the data\")\n",
+ " return (\n",
+ " client_data[\"users\"],\n",
+ " client_data[\"num_samples\"],\n",
+ " client_data[\"user_data\"],\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QBu1kiw8s-xr"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "jI3ixEN4s-xt",
+ "outputId": "ed969663-9e1e-4810-9507-52cdc426650a"
+ },
+ "source": [
+ "# From Femnist.py\n",
+ "for i in range(1):\n",
+ " cur_file = \"leaf/data/femnist/data/train/all_data_0_niid_0_keep_0_train_9.json\"\n",
+ " # test_file = \"leaf/data/femnist/data/test/all_data_0_niid_0_keep_0_test_9.json\"\n",
+ " # cur_file = test_file\n",
+ " clients, _, train_data = read_file(\n",
+ " os.path.join(train_dir, cur_file)\n",
+ " )\n",
+ " for cur_client in clients:\n",
+ " # self.clients.append(cur_client)\n",
+ " my_train_data[\"x\"].extend(train_data[cur_client][\"x\"])\n",
+ " my_train_data[\"y\"].extend(train_data[cur_client][\"y\"])\n",
+ " del train_data[cur_client]\n"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "wvHsSz8as-xw"
+ },
+ "source": [
+ "train_x = (\n",
+ " np.array(my_train_data[\"x\"], dtype=np.dtype(\"float32\"))\n",
+ " .reshape(-1, 28, 28, 1)\n",
+ " .transpose(0, 3, 1, 2)\n",
+ ")\n",
+ "train_y = np.array(my_train_data[\"y\"], dtype=np.dtype(\"int64\")).reshape(-1)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "K8X471SKs-xz",
+ "outputId": "cdf73c06-1323-4e76-850b-16324008d255"
+ },
+ "source": [
+ "len(train_y)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {
+ "id": "EpWNELBrs-x0"
+ },
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"wb\") as f:\n",
+ " pickle.dump({\"test_x\": train_x, \"test_y\": train_y}, f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "mAEASHr2s-x1"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "7665.166666666667"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "735856 / 96\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "id": "Am_XlcSSs-x3"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "id": "evAd9ZvYs-x6"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "id": "9_vIFakbs-x7",
+ "outputId": "3a8b546a-186f-4519-8c0b-e853986a8101"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(735856, 1, 28, 28)"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train[\"train_x\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "id": "GPyZ2C8ws-x9"
+ },
+ "outputs": [],
+ "source": [
+ "NUM_CLASSES = 62\n",
+ "IMAGE_SIZE = (28, 28)\n",
+ "FLAT_SIZE = 28 * 28\n",
+ "PIXEL_RANGE = 256.0\n",
+ "import torch.nn.functional as F\n",
+ "\n",
+ "class CNN(nn.Module):\n",
+ " \"\"\"\n",
+ " Class for a CNN Model for FEMNIST\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " def __init__(self):\n",
+ " \"\"\"\n",
+ " Constructor. Instantiates the CNN Model\n",
+ " with 28*28*1 Input and 62 output classes\n",
+ "\n",
+ " \"\"\"\n",
+ " super().__init__()\n",
+ " # 1.6 million params\n",
+ " self.conv1 = nn.Conv2d(1, 32, 5, padding=2)\n",
+ " self.pool = nn.MaxPool2d(2, 2)\n",
+ " self.conv2 = nn.Conv2d(32, 64, 5, padding=2)\n",
+ " self.fc1 = nn.Linear(7 * 7 * 64, 512)\n",
+ " self.fc2 = nn.Linear(512, NUM_CLASSES)\n",
+ "\n",
+ " def forward(self, x):\n",
+ " \"\"\"\n",
+ " Forward pass of the model\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " x : torch.tensor\n",
+ " The input torch tensor\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " torch.tensor\n",
+ " The output torch tensor\n",
+ "\n",
+ " \"\"\"\n",
+ " x = self.pool(F.relu(self.conv1(x)))\n",
+ " x = self.pool(F.relu(self.conv2(x)))\n",
+ " x = torch.flatten(x, 1)\n",
+ " x = F.relu(self.fc1(x))\n",
+ " x = self.fc2(x)\n",
+ " return x\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "id": "bCgW8ClBs-x_"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(735856,)"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train[\"train_y\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "id": "oBGwcwZks-yA"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from torch.utils.data import Dataset\n",
+ "\n",
+ "class FemnistDataset(Dataset):\n",
+ " def __init__(self, training, transform=None, target_transform=None):\n",
+ " if training:\n",
+ " with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " train = pickle.load(f)\n",
+ " self.data = train[\"train_x\"][10000:10000+7665,...]\n",
+ " self.label = train[\"train_y\"][10000:10000+7665,...]\n",
+ " else: \n",
+ " with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " test = pickle.load(f)\n",
+ " self.data = test[\"test_x\"]\n",
+ " self.label = test[\"test_y\"]\n",
+ " self.transform = transform\n",
+ " self.target_transform = target_transform\n",
+ "\n",
+ " def __len__(self):\n",
+ " return len(self.label)\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " return self.data[idx], self.label[idx]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "id": "U3boC_N4s-yC"
+ },
+ "outputs": [],
+ "source": [
+ "from torch.utils.data import DataLoader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "id": "sJsrQXkEs-yD"
+ },
+ "outputs": [],
+ "source": [
+ "trainset = FemnistDataset(True)\n",
+ "testset = FemnistDataset(False)\n",
+ "\n",
+ "train_dataloader = DataLoader(trainset, batch_size=16, shuffle=True)\n",
+ "test_dataloader = DataLoader(testset, batch_size=128, shuffle=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "480"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(train_dataloader)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "id": "e65Izyv0s-yE"
+ },
+ "outputs": [],
+ "source": [
+ "lr = 0.001\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1487, -0.1003, 0.0990, -0.0245, -0.1023, 0.0974, -0.1139, -0.1425,\n",
+ " -0.1949, -0.0679, -0.0937, 0.0891, 0.0577, -0.1357, 0.0814, 0.1157,\n",
+ " -0.1997, -0.1665, -0.1546, 0.1150, 0.0895, -0.1049, -0.0980, -0.0980,\n",
+ " 0.0729, 0.1947, 0.0421, -0.0365, -0.1470, -0.1679, 0.0286, -0.0146])"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(model.state_dict().values())[1]"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "tensor([-0.0180, 0.0236, 0.1279, -0.1352, -0.1948, -0.0330, -0.1615, -0.0286,\n",
+ " -0.1762, 0.0040, 0.1570, -0.1069, -0.1074, -0.1417, -0.1171, 0.0359,\n",
+ " 0.1276, -0.1534, -0.1773, -0.1639, 0.1334, 0.0518, 0.0586, 0.1466,\n",
+ " 0.1283, 0.0443, -0.0982, -0.1739, -0.0061, 0.1047, -0.0291, 0.1525])"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "for p in model.parameters():\n",
+ " print(p.data.size())\n",
+ " p.data = torch.zeros(p.data.size())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.1487, -0.1003, 0.0990, -0.0245, -0.1023, 0.0974, -0.1139, -0.1425,\n",
+ " -0.1949, -0.0679, -0.0937, 0.0891, 0.0577, -0.1357, 0.0814, 0.1157,\n",
+ " -0.1997, -0.1665, -0.1546, 0.1150, 0.0895, -0.1049, -0.0980, -0.0980,\n",
+ " 0.0729, 0.1947, 0.0421, -0.0365, -0.1470, -0.1679, 0.0286, -0.0146])"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(model.state_dict().values())[1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "eqOXilqMs-yF",
+ "outputId": "06799a3b-983b-4f51-a7bd-a901c041bd05"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.158939 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 34.5%, Avg loss: 2.492351 \n",
+ "\n",
+ "loss: 2.274407 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 49.2%, Avg loss: 2.004063 \n",
+ "\n",
+ "loss: 2.080229 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 57.7%, Avg loss: 1.550052 \n",
+ "\n",
+ "loss: 1.220055 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 62.5%, Avg loss: 1.387109 \n",
+ "\n",
+ "loss: 0.547404 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 62.6%, Avg loss: 1.411219 \n",
+ "\n",
+ "loss: 0.666172 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 68.1%, Avg loss: 1.147880 \n",
+ "\n",
+ "loss: 0.539106 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 66.9%, Avg loss: 1.218418 \n",
+ "\n",
+ "loss: 1.057546 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 68.9%, Avg loss: 1.211012 \n",
+ "\n",
+ "loss: 0.315841 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 66.5%, Avg loss: 1.400047 \n",
+ "\n",
+ "loss: 0.659244 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 66.0%, Avg loss: 1.484381 \n",
+ "\n",
+ "loss: 0.437452 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 69.8%, Avg loss: 1.239514 \n",
+ "\n",
+ "loss: 0.675393 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 69.3%, Avg loss: 1.224045 \n",
+ "\n",
+ "loss: 0.409850 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 67.5%, Avg loss: 1.499410 \n",
+ "\n",
+ "loss: 0.942130 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 69.7%, Avg loss: 1.331600 \n",
+ "\n",
+ "loss: 0.193678 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 69.5%, Avg loss: 1.398448 \n",
+ "\n",
+ "loss: 0.120872 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 68.4%, Avg loss: 1.589930 \n",
+ "\n",
+ "loss: 0.099591 [ 0/ 7665]\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(20):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " #print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " #print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " #print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lr = 0.001\n",
+ "model = CNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=0.001)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.158939 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 26.3%, Avg loss: 2.828093 \n",
+ "\n",
+ "loss: 2.478634 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 47.1%, Avg loss: 2.069258 \n",
+ "\n",
+ "loss: 2.099130 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 56.7%, Avg loss: 1.571264 \n",
+ "\n",
+ "loss: 1.343866 [ 0/ 7665]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 61.1%, Avg loss: 1.432287 \n",
+ "\n",
+ "loss: 0.783433 [ 0/ 7665]\n"
+ ]
+ },
+ {
+ "ename": "KeyboardInterrupt",
+ "evalue": "",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
+ "Input \u001b[0;32mIn [18]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 48\u001b[0m X \u001b[38;5;241m=\u001b[39m X\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m 49\u001b[0m y \u001b[38;5;241m=\u001b[39m y\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m---> 50\u001b[0m pred \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 51\u001b[0m test_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m loss_fn(pred, y)\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m 52\u001b[0m correct \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m (pred\u001b[38;5;241m.\u001b[39margmax(\u001b[38;5;241m1\u001b[39m) \u001b[38;5;241m==\u001b[39m y)\u001b[38;5;241m.\u001b[39mtype(torch\u001b[38;5;241m.\u001b[39mfloat)\u001b[38;5;241m.\u001b[39msum()\u001b[38;5;241m.\u001b[39mitem()\n",
+ "File \u001b[0;32m~/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/nn/modules/module.py:1102\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1098\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1099\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1100\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1101\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1103\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1104\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+ "Input \u001b[0;32mIn [11]\u001b[0m, in \u001b[0;36mCNN.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 29\u001b[0m \u001b[38;5;124;03mForward pass of the model\u001b[39;00m\n\u001b[1;32m 30\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 40\u001b[0m \n\u001b[1;32m 41\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 42\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool(F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv1(x)))\n\u001b[0;32m---> 43\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool(F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv2\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m))\n\u001b[1;32m 44\u001b[0m x \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mflatten(x, \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 45\u001b[0m x \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc1(x))\n",
+ "File \u001b[0;32m~/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/nn/modules/module.py:1102\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1098\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1099\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1100\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1101\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1103\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1104\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+ "File \u001b[0;32m~/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/nn/modules/conv.py:446\u001b[0m, in \u001b[0;36mConv2d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conv_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[0;32m~/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/nn/modules/conv.py:442\u001b[0m, in \u001b[0;36mConv2d._conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m 438\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mzeros\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m 439\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv2d(F\u001b[38;5;241m.\u001b[39mpad(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode),\n\u001b[1;32m 440\u001b[0m weight, bias, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstride,\n\u001b[1;32m 441\u001b[0m _pair(\u001b[38;5;241m0\u001b[39m), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdilation, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups)\n\u001b[0;32m--> 442\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv2d\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 443\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdilation\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\u001b[43m)\u001b[49m\n",
+ "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "for e in range(20):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " #print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " loss = loss_fn(pred, y)\n",
+ " # https://stackoverflow.com/questions/42704283/adding-l1-l2-regularization-in-pytorch\n",
+ " #to_cat = []\n",
+ " #for v in model.state_dict.values():\n",
+ " # flat = v.flatten()\n",
+ " # to_cat.append(flat)\n",
+ " #flat = torch.cat(to_cat, dim=0)\n",
+ " #loss = loss_fn(pred,y) + 0.02*torch.norm(flat, 2)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " #print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " optimizer.step()\n",
+ " optimizer.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " #print(optimizer.state.values())\n",
+ "\n",
+ " if batch % 500 == 0:\n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ " batch += 1\n",
+ " \n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "after 8: Accuracy: 67.3%, Avg loss: 1.200284 "
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "name": "learningrate.ipynb",
+ "provenance": []
+ },
+ "interpreter": {
+ "hash": "996934296aa9d79be6c3d800a38d8fdb7dfa8fe7bb07df178f1397cde2cb8742"
+ },
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/random files/multiprocessing_tests.py b/random files/multiprocessing_tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..59b00ba2276489160924366d749cf15ddb3e5d02
--- /dev/null
+++ b/random files/multiprocessing_tests.py
@@ -0,0 +1,56 @@
+import multiprocessing as omp
+import time
+from ctypes import c_int
+
+# from multiprocessing.queues import Queue
+from multiprocessing.sharedctypes import Value
+
+import torch.multiprocessing as mp
+
+
+def do_something_out(rank, node, queue, shared): # first argument is rank
+ print(f"run {rank}")
+ print(type(queue))
+ node.queue.put("do_something_out")
+ print(f"run {rank}")
+ node.val = 2
+ shared.value = 2
+ print("run")
+
+
+class Node:
+ def __init__(self, queue):
+ super(Node, self).__init__()
+ self.queue = queue
+ self.val = 1
+ # mp.spawn(Node.do_something)
+ # start_processes
+ # do_something_out(self)
+ self.x = Value(c_int, 1, lock=False)
+ print(type(self.x))
+ print("actual value:", self.x.value)
+ mp.start_processes(
+ do_something_out, args=(self, self.queue, self.x), start_method="fork"
+ )
+
+ def do_something(self):
+ # t.queue.put("do_something")
+ print("do_something")
+ self.val = 2
+
+
+if __name__ == "__main__":
+ queue = mp.Queue()
+ # queue.put("test")
+ node = Node(queue)
+ # mp.start_processes(
+ # do_something_out,
+ # (node, queue,),
+ # start_method="fork"
+ # )
+
+ time.sleep(3)
+ print(node.queue.get())
+ print(node.val)
+ print(node.x.value)
+ # print(node.val)
diff --git a/random files/playground_jw.ipynb b/random files/playground_jw.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..118dd638637b66c877d5927c5e4a9691116d86de
--- /dev/null
+++ b/random files/playground_jw.ipynb
@@ -0,0 +1,999 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from decentralizepy.datasets.Femnist import Femnist\n",
+ "from decentralizepy.graphs import SmallWorld\n",
+ "from collections import defaultdict\n",
+ "import os\n",
+ "import json\n",
+ "import numpy as np\n",
+ "import torch "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1256779281901044017"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "random_generator = torch.Generator()\n",
+ "# Will use the random device if supported by CPU, else uses the system time\n",
+ "# In the latter case we could get duplicate seeds on some of the machines\n",
+ "random_generator.seed()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "seed = random_generator.initial_seed()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0.7203, 0.5044, 0.1496, 0.6218, 0.5369, 0.8994, 0.9816, 0.9307, 0.7464,\n",
+ " 0.8109, 0.0372, 0.5295, 0.1905, 0.1923, 0.6036, 0.0288, 0.1395, 0.8307,\n",
+ " 0.0180, 0.8481])"
+ ]
+ },
+ "execution_count": 29,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.rand(20, generator = random_generator)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0.2485, 0.6399, 0.7028, 0.0918, 0.3800, 0.3301, 0.4674, 0.1821, 0.0140,\n",
+ " 0.5983, 0.4028, 0.2738, 0.5664, 0.1826, 0.8313, 0.8182, 0.8255, 0.4354,\n",
+ " 0.2829, 0.1707])"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.rand(20, generator = random_generator)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<torch._C.Generator at 0x7f415f3a9790>"
+ ]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "random_generator2 = torch.Generator()\n",
+ "random_generator2.manual_seed(seed)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0.7203, 0.5044, 0.1496, 0.6218, 0.5369, 0.8994, 0.9816, 0.9307, 0.7464,\n",
+ " 0.8109, 0.0372, 0.5295, 0.1905, 0.1923, 0.6036, 0.0288, 0.1395, 0.8307,\n",
+ " 0.0180, 0.8481])"
+ ]
+ },
+ "execution_count": 32,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.rand(20, generator = random_generator2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0.2485, 0.6399, 0.7028, 0.0918, 0.3800, 0.3301, 0.4674, 0.1821, 0.0140,\n",
+ " 0.5983, 0.4028, 0.2738, 0.5664, 0.1826, 0.8313, 0.8182, 0.8255, 0.4354,\n",
+ " 0.2829, 0.1707])"
+ ]
+ },
+ "execution_count": 33,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.rand(20, generator = random_generator2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 133,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0],\n",
+ " dtype=torch.int32)"
+ ]
+ },
+ "execution_count": 133,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Or we could use torch.bernoulli\n",
+ "alpha = 0.3\n",
+ "(torch.rand(size=(20,), generator = random_generator) < alpha).int()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 134,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "concated = torch.abs(torch.cat([torch.ones(10), torch.ones(5)*2], dim=0))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 135,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 2., 2., 2., 2., 2.])"
+ ]
+ },
+ "execution_count": 135,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "concated"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 136,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "15"
+ ]
+ },
+ "execution_count": 136,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "concated.size(dim = 0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 137,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1], dtype=torch.int32)"
+ ]
+ },
+ "execution_count": 137,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "binary_mask = (torch.rand(size=(concated.size(dim = 0),), generator=random_generator) < alpha).int()\n",
+ "binary_mask"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 138,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ True, True, False, False, False, False, True, False, False, False,\n",
+ " False, True, True, False, True])"
+ ]
+ },
+ "execution_count": 138,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "binary_mask = (torch.rand(size=(concated.size(dim = 0),), generator=random_generator) < alpha)\n",
+ "binary_mask"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 139,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([1., 1., 1., 2., 2., 2.])"
+ ]
+ },
+ "execution_count": 139,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "subsample = concated[binary_mask] # torch.masked_select\n",
+ "subsample"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 145,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "6"
+ ]
+ },
+ "execution_count": 145,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "subsample.size(dim = 0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 148,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
+ ]
+ },
+ "execution_count": 148,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "ground = torch.zeros(size = (15, ))\n",
+ "ground"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 143,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ True, True, False, False, False, False, True, False, False, False,\n",
+ " False, True, True, False, True])"
+ ]
+ },
+ "execution_count": 143,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "binary_mask"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 149,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0., 0., 0., 0., 0., 0.])"
+ ]
+ },
+ "execution_count": 149,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "ground[binary_mask]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 150,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ground[binary_mask] = subsample"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 151,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([1., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 2., 2., 0., 2.])"
+ ]
+ },
+ "execution_count": 151,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "ground"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Testing Serialization Speeds"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pickle\n",
+ "import time \n",
+ "import json\n",
+ "import ujson\n",
+ "import torch\n",
+ "import orjson\n",
+ "import numpy as np"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.009081792831420899\n"
+ ]
+ }
+ ],
+ "source": [
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " bin = pickle.dumps(test)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2.0366546154022216\n"
+ ]
+ }
+ ],
+ "source": [
+ "#single json encoding\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " test_list = test.numpy().tolist()\n",
+ " binary = json.dumps(test_list)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "str"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "type(binary)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2.500225853919983\n"
+ ]
+ }
+ ],
+ "source": [
+ "# double json encoding\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " test_list = test.numpy().tolist()\n",
+ " jsondata = json.dumps(test_list)\n",
+ " m = dict()\n",
+ " m[\"asjson\"] = jsondata\n",
+ " finaljson = json.dumps(m)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.7114214897155762\n"
+ ]
+ }
+ ],
+ "source": [
+ "#single ujson encoding: with to list\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " test_list = test.numpy().tolist()\n",
+ " binary = ujson.dumps(test_list)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "TypeError",
+ "evalue": "array([0.9719833 , 0.19759083, 0.06048423, ..., 0.10139698, 0.10623038,\n 0.16747206], dtype=float32) is not JSON serializable",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
+ "Input \u001b[0;32mIn [14]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m t1 \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[1;32m 6\u001b[0m test_list \u001b[38;5;241m=\u001b[39m test\u001b[38;5;241m.\u001b[39mnumpy()\u001b[38;5;66;03m#.tolist()\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m binary \u001b[38;5;241m=\u001b[39m \u001b[43mujson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdumps\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtest_list\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 8\u001b[0m times\u001b[38;5;241m.\u001b[39mappend(time\u001b[38;5;241m.\u001b[39mtime() \u001b[38;5;241m-\u001b[39m t1)\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28mprint\u001b[39m(np\u001b[38;5;241m.\u001b[39mmean(times))\n",
+ "\u001b[0;31mTypeError\u001b[0m: array([0.9719833 , 0.19759083, 0.06048423, ..., 0.10139698, 0.10623038,\n 0.16747206], dtype=float32) is not JSON serializable"
+ ]
+ }
+ ],
+ "source": [
+ "#single ujson encoding\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " test_list = test.numpy()#.tolist()\n",
+ " binary = ujson.dumps(test_list)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.31579806804656985\n"
+ ]
+ }
+ ],
+ "source": [
+ "#single orjson encoding: with to list\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " test_list = test.numpy().tolist()\n",
+ " binary = orjson.dumps(test_list)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.3235017776489258\n"
+ ]
+ }
+ ],
+ "source": [
+ "#single orjson encoding: with to list\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " test_list = test.numpy().tolist()\n",
+ " binary = orjson.dumps(test_list)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "bytes"
+ ]
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "type(binary)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.16293561458587646\n"
+ ]
+ }
+ ],
+ "source": [
+ "#single orjson encoding\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " test_list = test.numpy()\n",
+ " binary = orjson.dumps(test_list, option = orjson.OPT_SERIALIZE_NUMPY)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.40661447048187255\n"
+ ]
+ }
+ ],
+ "source": [
+ "# testing decode\n",
+ "#single orjson encoding: with to list\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " test_list = test.numpy()\n",
+ " binary = orjson.dumps(test_list, option = orjson.OPT_SERIALIZE_NUMPY)\n",
+ " decoded = orjson.loads(binary)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "list"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "type(decoded)"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "#single orjson encoding: with pickle\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " binary = pickle.dumps(test)\n",
+ " dictionary = {}\n",
+ " dictionary[\"array\"] = binary\n",
+ " json_binary = orjson.dumps(dictionary, option = orjson.OPT_SERIALIZE_NUMPY)\n",
+ " #decoded = orjson.loads(json_binary)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))\n",
+ "# TypeError: Type is not JSON serializable: bytes"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " binary = pickle.dumps(test)\n",
+ " m = dict()\n",
+ " m[\"seed\"] = 10\n",
+ " m[\"data\"] = binary\n",
+ " jsondump = json.dumps(m)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))\n",
+ "TypeError: Object of type bytes is not JSON serializable"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " binary = pickle.dumps(test)\n",
+ " m = dict()\n",
+ " m[\"seed\"] = 10\n",
+ " m[\"data\"] = binary.decode('utf-8')\n",
+ " jsondump = json.dumps(m)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))\n",
+ "# UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import base64"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 119,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.09808683395385742\n"
+ ]
+ }
+ ],
+ "source": [
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " binary = pickle.dumps(test)\n",
+ " m = dict()\n",
+ " m[\"seed\"] = 10\n",
+ " m[\"data\"] = base64.b64encode(binary).decode('utf-8')\n",
+ " jsondump = json.dumps(m)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 120,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'gASVTAAAAAAAAACMDHRvcmNoLl91dGlsc5SMEl9yZWJ1aWxkX3RlbnNvcl92MpSTlCiMDXRvcmNoLnN0b3JhZ2WUjBBfbG9hZF9m'"
+ ]
+ },
+ "execution_count": 120,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "m[\"data\"][0:100]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 121,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0.0484, 0.4790, 0.4148, 0.8002, 0.2449, 0.2223, 0.4366, 0.7714, 0.8805,\n",
+ " 0.7945, 0.6266, 0.1414, 0.1102, 0.9152, 0.3609, 0.0438, 0.1110, 0.9004,\n",
+ " 0.8536, 0.9587])"
+ ]
+ },
+ "execution_count": 121,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "test[0:20]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 122,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "seed = m[\"seed\"]\n",
+ "\n",
+ "params = m[\"data\"]\n",
+ "\n",
+ "\n",
+ "binary = base64.b64decode(params)\n",
+ "params = pickle.loads(binary)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 123,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0.0484, 0.4790, 0.4148, 0.8002, 0.2449, 0.2223, 0.4366, 0.7714, 0.8805,\n",
+ " 0.7945, 0.6266, 0.1414, 0.1102, 0.9152, 0.3609, 0.0438, 0.1110, 0.9004,\n",
+ " 0.8536, 0.9587])"
+ ]
+ },
+ "execution_count": 123,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "params[0:20]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.05851750373840332\n"
+ ]
+ }
+ ],
+ "source": [
+ "# with orjson\n",
+ "times = []\n",
+ "for i in range(10):\n",
+ " test = torch.rand(size = (5000000,))\n",
+ " t1 = time.time()\n",
+ " binary = pickle.dumps(test)\n",
+ " m = dict()\n",
+ " m[\"seed\"] = 10\n",
+ " m[\"data\"] = base64.b64encode(binary).decode('utf-8')\n",
+ " jsondump = orjson.dumps(m)\n",
+ " times.append(time.time() - t1)\n",
+ "print(np.mean(times))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Code Snipets from Testing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 159,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9],\n",
+ " [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]])"
+ ]
+ },
+ "execution_count": 159,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.arange(0,20,1).reshape([2,10])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 160,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,\n",
+ " 18, 19])"
+ ]
+ },
+ "execution_count": 160,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.arange(0,20,1).reshape([2,10]).flatten()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "interpreter": {
+ "hash": "996934296aa9d79be6c3d800a38d8fdb7dfa8fe7bb07df178f1397cde2cb8742"
+ },
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/random files/playground_jw_quant.ipynb b/random files/playground_jw_quant.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..bcdd39d09e3fd931e7a1ce9283b12850d48e4808
--- /dev/null
+++ b/random files/playground_jw_quant.ipynb
@@ -0,0 +1,4179 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from qtorch.quant import fixed_point_quantize, block_quantize, float_quantize"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from qtorch import FloatingPoint"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pickle"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 77,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "import fpzip\n",
+ "from pyzfp import compress, decompress\n",
+ "import torch\n",
+ "import matplotlib.pyplot as plt\n",
+ "from torch.quantization.observer import MinMaxObserver, MovingAverageMinMaxObserver, HistogramObserver\n",
+ "from torch.nn.quantized.modules.utils import _quantize_weight\n",
+ "import pywt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from decentralizepy.compression.Elias8bitQuant import Elias8bitQuant\n",
+ "ebit8q = Elias8bitQuant()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "a = np.random.normal(0, 0.1, 10000).astype(np.float32)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "40000"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(a)*4"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "b = fpzip.compress(a , precision=12, order='C')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "12087"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(b)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "c = fpzip.decompress(b, order='C')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([-0.01464844, -0.0859375 , -0.00268555, 0.01953125, 0.00317383,\n",
+ " -0.078125 , -0.109375 , 0.00585938, -0.1875 , -0.01464844],\n",
+ " dtype=float32)"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "c[0][0][0][0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0.50458467"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.linalg.norm((a-c)[0][0], 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([-0.01551323, -0.08976793, -0.00279709, ..., 0.0934156 ,\n",
+ " -0.00322201, -0.16196491], dtype=float32)"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "a"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "at = torch.from_numpy(a)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "stochastic_rounded = float_quantize(at, exp=3, man=5, rounding=\"stochastic\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "7.9375"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "float_quantize(torch.tensor(np.array([10]), dtype = torch.float32), exp=2, man=6, rounding=\"nearest\").item()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# something is off this should be half the size\n",
+ "# it is correct 2 bits can represent 0,1,2,3\n",
+ "# 3 - 1 = 2\n",
+ "# and 2^2 is 4\n",
+ "# They do no do subnormal numbers"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0.0078125"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "fixed_point_quantize(torch.tensor(np.array([0.01]), dtype = torch.float32), 8, 7, rounding=\"nearest\").item()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# there is 1 bit for sign"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0156, -0.0859, -0.0078, ..., 0.0938, 0.0000, -0.1641])"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "stochastic_rounded"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0156, -0.0859, -0.0078, ..., 0.0938, 0.0000, -0.1641])"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "stochastic_rounded"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "bytes_array = stochastic_rounded.numpy().tobytes()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "numpy.ndarray"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.ndarray"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([ 0, 0, 128, ..., 0, 40, 190], dtype=uint8)"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.frombuffer(bytes_array, dtype = np.uint8)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "40399"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(pickle.dumps(stochastic_rounded))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "36855"
+ ]
+ },
+ "execution_count": 28,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(pickle.dumps(fpzip.compress(stochastic_rounded.numpy())))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "FloatingPoint (exponent=3, mantissa=5)"
+ ]
+ },
+ "execution_count": 29,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "FloatingPoint(3,5)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "40399"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(pickle.dumps(at))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "37119"
+ ]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(pickle.dumps(fpzip.compress(at.numpy())))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "10000"
+ ]
+ },
+ "execution_count": 32,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(at)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "10000 torch.float32\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(len(stochastic_rounded), stochastic_rounded.dtype)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(0.3202)"
+ ]
+ },
+ "execution_count": 34,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(at - stochastic_rounded, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#backend = 'fbgemm' if x86 else 'qnnpack'\n",
+ "#qconfig = torch.quantization.get_default_qconfig(backend) \n",
+ "#torch.backends.quantized.engine = backend"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'fbgemm'"
+ ]
+ },
+ "execution_count": 36,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.backends.quantized.engine"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "MinMaxObserver (tensor([0.0031]), tensor([123], dtype=torch.int32))\n",
+ "MovingAverageMinMaxObserver (tensor([0.0031]), tensor([123], dtype=torch.int32))\n",
+ "HistogramObserver (tensor([0.0031]), tensor([123], dtype=torch.int32))\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/jeffrey/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/ao/quantization/observer.py:886: UserWarning: __floordiv__ is deprecated, and its behavior will change in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values. To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor').\n",
+ " src_bin_begin // dst_bin_width, 0, self.dst_nbins - 1\n",
+ "/home/jeffrey/anaconda3/envs/sacs39/lib/python3.9/site-packages/torch/ao/quantization/observer.py:891: UserWarning: __floordiv__ is deprecated, and its behavior will change in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values. To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor').\n",
+ " src_bin_end // dst_bin_width, 0, self.dst_nbins - 1\n"
+ ]
+ }
+ ],
+ "source": [
+ "a = np.random.normal(0, 0.1, 10000).astype(np.float32)\n",
+ "scheme = torch.per_tensor_affine#per_tensor_affine # affine means taking into account the actual range of the values\n",
+ "observers = [MinMaxObserver(qscheme=scheme), MovingAverageMinMaxObserver(qscheme=scheme), HistogramObserver(qscheme=scheme)]\n",
+ "for obs in observers:\n",
+ " #for x in inputs: obs(x) \n",
+ " obs.forward(torch.tensor(a))#(torch.tensor(a))\n",
+ " print(obs.__class__.__name__, obs.calculate_qparams())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "o = observers[0].calculate_qparams()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(tensor([0.0031]), tensor([123], dtype=torch.int32))"
+ ]
+ },
+ "execution_count": 39,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "o"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0.0031])"
+ ]
+ },
+ "execution_count": 40,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "o[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "aq = _quantize_weight(torch.tensor(a), observers[2])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0124, 0.0124, -0.0838, ..., -0.0217, -0.1644, 0.0124],\n",
+ " size=(10000,), dtype=torch.qint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.003101914655417204,\n",
+ " zero_point=123)"
+ ]
+ },
+ "execution_count": 42,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "aq"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0248, 0.0838, -0.0838, ..., -0.0217, -0.1645, 0.1738],\n",
+ " size=(10000,), dtype=torch.quint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.003103430150076747,\n",
+ " zero_point=123)"
+ ]
+ },
+ "execution_count": 43,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.quantize_per_tensor(\n",
+ " torch.tensor(a),\n",
+ " o[0].item(), o[1].item(), torch.quint8)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "qt = torch.quantize_per_tensor(\n",
+ " torch.tensor(a),\n",
+ " o[0].item(), o[1].item(), torch.quint8)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "qscheme=torch.qint8"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 46,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0124, 0.0124, -0.0838, ..., -0.0217, -0.1644, 0.0124],\n",
+ " size=(10000,), dtype=torch.qint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.003101914655417204,\n",
+ " zero_point=123)"
+ ]
+ },
+ "execution_count": 46,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "aq"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 47,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0.003101914655417204"
+ ]
+ },
+ "execution_count": 47,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "aq.q_scale()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "123"
+ ]
+ },
+ "execution_count": 48,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "aq.q_zero_point()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 49,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(127, dtype=torch.int8)"
+ ]
+ },
+ "execution_count": 49,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "aq.int_repr().max()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQCklEQVR4nO3df6zddX3H8edrFNCos/y4a1hbV4zNHH9MJA3DaBYHE/lhLEvQYMzosEmTDTONS1wdicZsS2BLZJIYXSNmZWECooZO2bTyI2Z/gBZFLKDjwiBtU2jllxqiG/reH+dTPNR7e8/91XvPh+cjOTmf7/v7Oef7+aT3vvrt5/s9p6kqJEl9+Y2lHoAkaeEZ7pLUIcNdkjpkuEtShwx3SeqQ4S5JHRop3JM8muT7Se5NsqvVTkyyM8lD7fmEVk+Sa5JMJrkvyRmLOQFJ0q+bzZn7H1XV6VW1oW1vBW6rqvXAbW0b4HxgfXtsAT69UIOVJI1mPssyG4Htrb0duGiofl0N3AWsTHLKPI4jSZqlFSP2K+DrSQr456raBqyqqv1t/+PAqtZeDewZeu3eVtvPNE4++eRat27dbMYtSS9599xzz4+qamKqfaOG+1uqal+S3wJ2JvnB8M6qqhb8I0uyhcGyDa95zWvYtWvXbF4uSS95SR6bbt9IyzJVta89HwC+DJwJPHFouaU9H2jd9wFrh16+ptUOf89tVbWhqjZMTEz5F48kaY5mDPckr0jyqkNt4FxgN7AD2NS6bQJuae0dwKXtrpmzgGeHlm8kSUfBKMsyq4AvJznU/9+q6j+TfBu4Kclm4DHg3a3/rcAFwCTwHHDZgo9aknREM4Z7VT0CvGGK+pPAOVPUC7h8QUYnSZoTP6EqSR0y3CWpQ4a7JHXIcJekDhnuktShUT+hKuklaN3Wr77QfvTKC5dwJJotz9wlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjo0crgnOSbJd5N8pW2fmuTuJJNJbkxyXKsf37Yn2/51izR2SdI0ZnPm/gHgwaHtq4Crq+p1wNPA5lbfDDzd6le3fpKko2ikcE+yBrgQ+GzbDnA2cHPrsh24qLU3tm3a/nNaf0nSUTLqmfs/AR8Gftm2TwKeqarn2/ZeYHVrrwb2ALT9z7b+kqSjZMZwT/IO4EBV3bOQB06yJcmuJLsOHjy4kG8tSS95o5y5vxl4Z5JHgRsYLMd8EliZZEXrswbY19r7gLUAbf+rgScPf9Oq2lZVG6pqw8TExLwmIUl6sRnDvao+UlVrqmodcAlwe1W9F7gDuLh12wTc0to72jZt/+1VVQs6aknSEc3nPve/Bj6UZJLBmvq1rX4tcFKrfwjYOr8hSpJma8XMXX6lqu4E7mztR4Azp+jzM+BdCzA2SdIc+QlVSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR2aMdyTvCzJt5J8L8n9ST7e6qcmuTvJZJIbkxzX6se37cm2f90iz0GSdJhRztx/DpxdVW8ATgfOS3IWcBVwdVW9Dnga2Nz6bwaebvWrWz9J0lE0Y7jXwE/b5rHtUcDZwM2tvh24qLU3tm3a/nOSZKEGLEma2Uhr7kmOSXIvcADYCTwMPFNVz7cue4HVrb0a2APQ9j8LnLSAY5YkzWCkcK+qX1TV6cAa4Ezg9fM9cJItSXYl2XXw4MH5vp0kacis7papqmeAO4A3ASuTrGi71gD7WnsfsBag7X818OQU77WtqjZU1YaJiYm5jV6SNKVR7paZSLKytV8OvA14kEHIX9y6bQJuae0dbZu2//aqqgUcsyRpBitm7sIpwPYkxzD4y+CmqvpKkgeAG5L8HfBd4NrW/1rgX5NMAk8BlyzCuCVJRzBjuFfVfcAbp6g/wmD9/fD6z4B3LcjoJElz4idUJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHVoxnBPsjbJHUkeSHJ/kg+0+olJdiZ5qD2f0OpJck2SyST3JTljsSchSXqxUc7cnwf+qqpOA84CLk9yGrAVuK2q1gO3tW2A84H17bEF+PSCj1qSdEQzhntV7a+q77T2T4AHgdXARmB767YduKi1NwLX1cBdwMokpyz0wCVJ05vVmnuSdcAbgbuBVVW1v+16HFjV2quBPUMv29tqkqSjZORwT/JK4IvAB6vqx8P7qqqAms2Bk2xJsivJroMHD87mpZKkGYwU7kmOZRDs11fVl1r5iUPLLe35QKvvA9YOvXxNq71IVW2rqg1VtWFiYmKu45ckTWGUu2UCXAs8WFWfGNq1A9jU2puAW4bql7a7Zs4Cnh1avpEkHQUrRujzZuBPge8nubfV/ga4ErgpyWbgMeDdbd+twAXAJPAccNlCDliSNLMZw72q/gvINLvPmaJ/AZfPc1ySpHnwE6qS1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOjRjuCf5XJIDSXYP1U5MsjPJQ+35hFZPkmuSTCa5L8kZizl4SdLURjlz/xfgvMNqW4Hbqmo9cFvbBjgfWN8eW4BPL8wwJUmzMWO4V9U3gacOK28Etrf2duCiofp1NXAXsDLJKQs0VknSiOa65r6qqva39uPAqtZeDewZ6re31SRJR9G8L6hWVQE129cl2ZJkV5JdBw8enO8wJElD5hruTxxabmnPB1p9H7B2qN+aVvs1VbWtqjZU1YaJiYk5DkOSNJW5hvsOYFNrbwJuGapf2u6aOQt4dmj5RpJ0lKyYqUOSzwNvBU5Oshf4GHAlcFOSzcBjwLtb91uBC4BJ4DngskUYsyRpBjOGe1W9Z5pd50zRt4DL5zsoSdL8+AlVSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1KEZb4WU1Kd1W7/6QvvRKy9cwpFoMXjmLkkdMtwlqUOGuyR1yDV3SS8yvBav8WW4Sy8hBvdLh8syktQhz9wleUbfIc/cJalDhrskdchlGalzLrm8NBnuUocMdLksI0kdMtwlqUOGuyR1yHCXpA55QVXqhBdRNcwzd0nqkOEuSR0y3CWpQ665S2PMdXZNx3CXxoyBrlG4LCNJHTLcJalDhrskdchwl6QOeUFVWqaGL5w+euWFSzgSjSPP3CWpQ565S8vIdLc5evujZmtRwj3JecAngWOAz1bVlYtxHKkHBrcWw4IvyyQ5BvgUcD5wGvCeJKct9HEkSdNbjDP3M4HJqnoEIMkNwEbggUU4ljSWPFvXYluMcF8N7Bna3gv8wSIcR1pS093NYnBrOViyC6pJtgBb2uZPk/xwjm91MvCjhRnVknMuy89I88hVR2Ek8zevP5NlNsdefr5gfnP5nel2LEa47wPWDm2vabUXqaptwLb5HizJrqraMN/3WQ6cy/LTyzzAuSxXizWXxbjP/dvA+iSnJjkOuATYsQjHkSRNY8HP3Kvq+STvB77G4FbIz1XV/Qt9HEnS9BZlzb2qbgVuXYz3nsK8l3aWEeey/PQyD3Auy9WizCVVtRjvK0laQn63jCR1aKzCPcnfJrkvyb1Jvp7kt1s9Sa5JMtn2nzH0mk1JHmqPTUs3+hdL8o9JftDG++UkK4f2faTN5YdJ3j5UP6/VJpNsXZKBHybJu5Lcn+SXSTYctm9s5jGVcRnnIUk+l+RAkt1DtROT7Gw//zuTnNDq0/7OLLUka5PckeSB9rP1gVYfx7m8LMm3knyvzeXjrX5qkrvbmG9sN5+Q5Pi2Pdn2r5vzwatqbB7Abw61/xL4TGtfAPwHEOAs4O5WPxF4pD2f0NonLPU82tjOBVa09lXAVa19GvA94HjgVOBhBhemj2nt1wLHtT6nLYN5/B7wu8CdwIah+ljNY4p5jcU4DxvzHwJnALuHav8AbG3trUM/Z1P+ziyHB3AKcEZrvwr47/bzNI5zCfDK1j4WuLuN8Sbgklb/DPDnrf0XQ7l2CXDjXI89VmfuVfXjoc1XAIcuGGwErquBu4CVSU4B3g7srKqnquppYCdw3lEd9DSq6utV9XzbvIvB5wFgMJcbqurnVfU/wCSDr3R44Wsdqup/gUNf67CkqurBqprqA2hjNY8pjMs4X1BV3wSeOqy8Edje2tuBi4bqU/3OLLmq2l9V32ntnwAPMvjk+zjOparqp23z2PYo4Gzg5lY/fC6H5ngzcE6SzOXYYxXuAEn+Pske4L3AR1t5qq88WH2E+nLzPgZnHjD+czlk3OcxLuOcyaqq2t/ajwOrWnss5teWJd7I4Ix3LOeS5Jgk9wIHGJxgPgw8M3RyNzzeF+bS9j8LnDSX4y67cE/yjSS7p3hsBKiqK6pqLXA98P6lHe2RzTSX1ucK4HkG81mWRpmHlr8a/Ft/bG6PS/JK4IvABw/7V/tYzaWqflFVpzP41/mZwOuPxnGX3X/WUVV/PGLX6xncS/8xpv/Kg33AWw+r3znvQY5oprkk+TPgHcA57YcVjvz1DTN+rcNimMWfybBlN49ZGulrNMbAE0lOqar9baniQKsv6/klOZZBsF9fVV9q5bGcyyFV9UySO4A3MVg6WtHOzofHe2gue5OsAF4NPDmX4y27M/cjSbJ+aHMj8IPW3gFc2q6anwU82/759jXg3CQntCvr57baksvgPzT5MPDOqnpuaNcO4JJ21fxUYD3wLcbvax3GfR7jMs6Z7AAO3SW2CbhlqD7V78ySa2vM1wIPVtUnhnaN41wm0u6ES/Jy4G0MriHcAVzcuh0+l0NzvBi4fejEb3aW+mrybB4M/ibfDdwH/Duwun51RfpTDNayvs+L79p4H4OLeZPAZUs9h6FxTTJYW7u3PT4ztO+KNpcfAucP1S9gcOfAw8AVSz2HNqY/YbBm+HPgCeBr4ziPaeY2FuMcGu/ngf3A/7U/k80M1mtvAx4CvgGc2PpO+zuz1A/gLQyWXO4b+v24YEzn8vvAd9tcdgMfbfXXMjjZmQS+ABzf6i9r25Nt/2vnemw/oSpJHRqrZRlJ0mgMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOvT/DCIxoj6P6T0AAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(aq.int_repr().to(torch.float32).numpy()[0:1000], 100, (-300, 300))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQeUlEQVR4nO3df6zddX3H8edLfojxx6By13RAV4wNjiwD2Q3DaIyjggjGsoQRjNk6JWmyzQ2zLa6OxMVsS2BLdC4xc424dQkTECXt3KbWDmKWTLQoIlAYhUGElLYq+DPRoe/9cT6F4+W099wf5977aZ+P5OR8v5/v9/S8P7n3vvLp5/srVYUkqT8vWO4CJEnzY4BLUqcMcEnqlAEuSZ0ywCWpU8cv5ZedeuqptW7duqX8Sknq3l133fXNqpqa2b6kAb5u3Tp27969lF8pSd1L8tiodqdQJKlTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpU0t6Jaakpbduy789u/zodZctYyVabI7AJalTjsAlOUrvlCNwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6NWuAJzkryd1Dr+8meXeSVUl2JnmovZ+yFAVLkgZmDfCqerCqzq2qc4FfBX4I3AZsAXZV1XpgV1uXJC2RuU6hbAAerqrHgI3Atta+Dbh8EeuSJM1irgF+FfDxtry6qva15SeB1YtWlSRpVmMHeJITgbcCn5i5raoKqMN8bnOS3Ul2Hzx4cN6FSpJ+1lxG4G8GvlJV+9v6/iRrANr7gVEfqqqtVTVdVdNTU1MLq1aS9Ky5BPjbeG76BGAHsKktbwK2L1ZRkqTZjRXgSV4MXAR8aqj5OuCiJA8Bb2zrkqQlMtb9wKvqB8DLZ7R9i8FZKZKkZeCVmJLUKQNckjplgEtSp3wmpnSMGn4O5jj7+KzMlccRuCR1ygCXpE4Z4JLUKQNckjrlQUzpKDTOAUr1zxG4JHXKAJekTjmFImksnhO+8jgCl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpU+M+1PjkJLcmeSDJniSvSbIqyc4kD7X3UyZdrCTpOeOOwD8EfKaqXgWcA+wBtgC7qmo9sKutS5KWyKwBnuTngNcDNwBU1Y+r6mlgI7Ct7bYNuHwyJUqSRhnnUvozgYPAPyY5B7gLuAZYXVX72j5PAqtHfTjJZmAzwNq1axdcsKT5m+tj1LSyjTOFcjxwHvD3VfVq4AfMmC6pqgJq1IeramtVTVfV9NTU1ELrlSQ14wT448DjVXVnW7+VQaDvT7IGoL0fmEyJkqRRZg3wqnoS+EaSs1rTBuB+YAewqbVtArZPpEJJ0kjj3k72D4Abk5wIPAK8g0H435LkauAx4MrJlChJGmWsAK+qu4HpEZs2LGo1kqSxeSWmJHXKAJekTvlINUlz5uPVVgZH4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ3yboTSUcKnyR97HIFLUqcMcEnq1FhTKEkeBb4H/AR4pqqmk6wCbgbWAY8CV1bVU5MpU5I001xG4L9eVedW1aGHG28BdlXVemBXW5ckLZGFTKFsBLa15W3A5QuuRpI0tnHPQingc0kK+Ieq2gqsrqp9bfuTwOpRH0yyGdgMsHbt2gWWK2mYZ54c28YN8NdV1RNJfh7YmeSB4Y1VVS3cn6eF/VaA6enpkftIkuZurCmUqnqivR8AbgPOB/YnWQPQ3g9MqkhJ0vPNGuBJXpzkpYeWgYuBe4EdwKa22yZg+6SKlCQ93zhTKKuB25Ic2v9fquozSb4M3JLkauAx4MrJlSlJmmnWAK+qR4BzRrR/C9gwiaIkSbPzSkxJ6pQBLkmdMsAlqVMGuCR1ygCXpE75QAdJCzJ8Of+j1122jJUcexyBS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTYwd4kuOSfDXJp9v6mUnuTLI3yc1JTpxcmZKkmeYyAr8G2DO0fj3wwap6JfAUcPViFiZJOrKxAjzJ6cBlwEfbeoALgVvbLtuAyydQnyTpMMYdgf8t8B7gp2395cDTVfVMW38cOG3UB5NsTrI7ye6DBw8upFZJ0pBZAzzJW4ADVXXXfL6gqrZW1XRVTU9NTc3nn5AkjTDOE3leC7w1yaXAScDLgA8BJyc5vo3CTweemFyZkqSZZg3wqnov8F6AJG8A/qSq3p7kE8AVwE3AJmD75MqUdMjwI8x0bFvIeeB/CvxRkr0M5sRvWJySJEnjmNNDjavqDuCOtvwIcP7ilyRJGodXYkpSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekTs3pdrKSdCTDD5t49LrLlrGSY4MjcEnqlAEuSZ0ywCWpUwa4JHVq1gBPclKSLyX5WpL7kry/tZ+Z5M4ke5PcnOTEyZcrSTpknBH4j4ALq+oc4FzgkiQXANcDH6yqVwJPAVdPrEpJ0vPMGuA18P22ekJ7FXAhcGtr3wZcPokCJUmjjXUeeJLjgLuAVwIfBh4Gnq6qZ9oujwOnHeazm4HNAGvXrl1ovdIxafj8aumQsQ5iVtVPqupc4HTgfOBV435BVW2tqumqmp6amppflZKk55nTWShV9TRwO/Aa4OQkh0bwpwNPLG5pkqQjGecslKkkJ7flFwEXAXsYBPkVbbdNwPYJ1ShJGmGcOfA1wLY2D/4C4Jaq+nSS+4Gbkvwl8FXghgnWKUmaYdYAr6p7gFePaH+EwXy4JGkZeCWmJHXKAJekTnk/cGmF8txvzcYRuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqe8G6GkiRi+m+Kj1122jJUcvRyBS1Knxnmo8RlJbk9yf5L7klzT2lcl2ZnkofZ+yuTLlSQdMs4I/Bngj6vqbOAC4PeTnA1sAXZV1XpgV1uXJC2RWQO8qvZV1Vfa8veAPcBpwEZgW9ttG3D5hGqUJI0wpznwJOsYPKH+TmB1Ve1rm54EVi9uaZKkIxk7wJO8BPgk8O6q+u7wtqoqoA7zuc1JdifZffDgwQUVK0l6zlgBnuQEBuF9Y1V9qjXvT7KmbV8DHBj12araWlXTVTU9NTW1GDVLkhjvLJQANwB7quoDQ5t2AJva8iZg++KXJ0k6nHEu5Hkt8FvA15Pc3dr+DLgOuCXJ1cBjwJUTqVCSNNKsAV5V/wXkMJs3LG45kqRxeSm9pIkbvqwevLR+sXgpvSR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnfI8cGmZ+egxzZcjcEnqlAEuSZ1yCkXSknPaaHE4ApekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVOzngee5GPAW4ADVfXLrW0VcDOwDngUuLKqnppcmdLRZeYjxmZrl0YZZwT+T8AlM9q2ALuqaj2wq61LkpbQrAFeVV8Avj2jeSOwrS1vAy5f3LIkSbOZ7xz46qra15afBFYfbsckm5PsTrL74MGD8/w6SdJMCz6IWVUF1BG2b62q6aqanpqaWujXSZKa+Qb4/iRrANr7gcUrSZI0jvnejXAHsAm4rr1vX7SKpKOUZ5hosc06Ak/yceC/gbOSPJ7kagbBfVGSh4A3tnVJ0hKadQReVW87zKYNi1yLJGkOfKCDpGXlwx3mz0vpJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlOeBS1oxDne7Ac8PH80RuCR1ygCXpE45hSJNkHcg1CQ5ApekThngktQpp1CkBfJueloujsAlqVOOwKVF5EHLyfD88NEcgUtSpwxwSerUgqZQklwCfAg4DvhoVflwYx1VDneA0qmSleFYP4A87xF4kuOADwNvBs4G3pbk7MUqTJJ0ZAuZQjkf2FtVj1TVj4GbgI2LU5YkaTYLmUI5DfjG0PrjwK/N3CnJZmBzW/1+kgfn+X2nAt+c52dXmqOlL0dLP2CMvuT6Japk4Y6pn8shK/zns9CfyS+Oapz4aYRVtRXYutB/J8nuqppehJKW3dHSl6OlH2BfVqqjpS+T6sdCplCeAM4YWj+9tUmSlsBCAvzLwPokZyY5EbgK2LE4ZUmSZjPvKZSqeibJu4DPMjiN8GNVdd+iVfZ8C56GWUGOlr4cLf0A+7JSHS19mUg/UlWT+HclSRPmlZiS1CkDXJI6tSIDPMlfJLknyd1JPpfkF1p7kvxdkr1t+3lDn9mU5KH22rR81T8nyd8keaDVeluSk4e2vbf148Ekbxpqv6S17U2yZVkKHyHJbya5L8lPk0zP2NZVX2bqpc5DknwsyYEk9w61rUqys/3+70xySms/7N/McktyRpLbk9zffreuae099uWkJF9K8rXWl/e39jOT3Nlqvrmd8EGSF7b1vW37unl9cVWtuBfwsqHlPwQ+0pYvBf4DCHABcGdrXwU80t5PacunrIB+XAwc35avB65vy2cDXwNeCJwJPMzgQPBxbfkVwIltn7OXux+t5l8CzgLuAKaH2rvry4x+dVHnjJpfD5wH3DvU9tfAlra8Zeh3beTfzEp4AWuA89ryS4H/ab9PPfYlwEva8gnAna3GW4CrWvtHgN9ty783lGtXATfP53tX5Ai8qr47tPpi4NCR1o3AP9fAF4GTk6wB3gTsrKpvV9VTwE7gkiUteoSq+lxVPdNWv8jgXHkY9OOmqvpRVf0vsJfBrQlW7O0JqmpPVY26ira7vszQS53PqqovAN+e0bwR2NaWtwGXD7WP+ptZdlW1r6q+0pa/B+xhcIV3j32pqvp+Wz2hvQq4ELi1tc/sy6E+3gpsSJK5fu+KDHCAJH+V5BvA24H3teZRl++fdoT2leSdDEYP0Hc/Zuq9L73UOZvVVbWvLT8JrG7LXfSvTSG8msHItcu+JDkuyd3AAQaDyIeBp4cGccP1PtuXtv07wMvn+p3LFuBJPp/k3hGvjQBVdW1VnQHcCLxrueqczWz9aPtcCzzDoC8r1jh90cpXg/+Xd3N+cJKXAJ8E3j3jf99d9aWqflJV5zL4n/b5wKsm/Z3L9ki1qnrjmLveCPw78Occ/vL9J4A3zGi/Y8FFjmG2fiT5HeAtwIb2ywhHvg3Bst2eYA4/k2Ersi9zcLTcEmJ/kjVVta9NKxxo7Su6f0lOYBDeN1bVp1pzl305pKqeTnI78BoG0zzHt1H2cL2H+vJ4kuOBnwO+NdfvWpFTKEnWD61uBB5oyzuA325Hoy8AvtP+q/VZ4OIkp7Qj1he3tmWVwQMv3gO8tap+OLRpB3BVOxJ9JrAe+BJ93p6g9770UudsdgCHzr7aBGwfah/1N7Ps2pzvDcCeqvrA0KYe+zKVdpZZkhcBFzGY078duKLtNrMvh/p4BfCfQwO88S330dvDHNH9JHAvcA/wr8BpQ0d6P8xgbunr/OzZEO9kcABtL/CO5e5Dq2kvg3muu9vrI0Pbrm39eBB481D7pQyOxj8MXLvcfRiq6zcYzOH9CNgPfLbXvozoWxd1DtX7cWAf8H/tZ3I1g/nTXcBDwOeBVW3fw/7NLPcLeB2D6ZF7hv5GLu20L78CfLX15V7gfa39FQwGNHuBTwAvbO0ntfW9bfsr5vO9XkovSZ1akVMokqTZGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpU/8PAAk5F59lVssAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(qt.int_repr().to(torch.float32).numpy()[0:1000], 100, (-300, 300))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def convert(aq):\n",
+ " return (aq.int_repr().to(torch.float32) - aq.q_zero_point())*aq.q_scale()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor([ 0.0124, 0.0124, -0.0838, ..., -0.0217, -0.1644, 0.0124],\n",
+ " size=(10000,), dtype=torch.qint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.003101914655417204,\n",
+ " zero_point=123)\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(aq)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([ 0.0124, 0.0124, -0.0838, ..., -0.0217, -0.1644, 0.0124])"
+ ]
+ },
+ "execution_count": 54,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "convert(aq)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([127, 127, 96, ..., 116, 70, 127], dtype=torch.int8)"
+ ]
+ },
+ "execution_count": 55,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "aq.int_repr()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "a = np.random.rand(100).astype(np.float32)\n",
+ "at = torch.from_numpy(a)\n",
+ "scheme = torch.per_tensor_affine\n",
+ "observer = MovingAverageMinMaxObserver(qscheme=scheme) #MinMaxObserver, MovingAverageMinMaxObserver(qscheme=scheme), HistogramObserver(qscheme=scheme)]\n",
+ "observer.forward(at)\n",
+ "o = observer.calculate_qparams()\n",
+ "aq = torch.quantize_per_tensor(\n",
+ " torch.tensor(a),\n",
+ " o[0].item(), o[1].item(), torch.quint8)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0.010789979249238968"
+ ]
+ },
+ "execution_count": 57,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(at - convert(aq), 2).item()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "comp = ebit8q.compress_float(a)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 59,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "decomp = ebit8q.decompress_float(comp)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 60,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0.010789979249238968"
+ ]
+ },
+ "execution_count": 60,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(at - decomp, 2).item()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 110,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def comp(a):\n",
+ " b = fpzip.compress(a , precision=14, order='C')\n",
+ " c = fpzip.decompress(b, order='C')\n",
+ " bb = compress(a, tolerance=0.001, parallel=False)\n",
+ " cc = decompress(bb, a.shape, a.dtype, tolerance=0.001)\n",
+ " \n",
+ " at = torch.from_numpy(a)\n",
+ " stochastic_rounded = float_quantize(at, exp=5, man=2, rounding=\"stochastic\")\n",
+ " \n",
+ " scheme = torch.per_tensor_affine#per_tensor_affine # affine means taking into account the actual range of the values\n",
+ " observer = MovingAverageMinMaxObserver(qscheme=scheme) #MinMaxObserver, MovingAverageMinMaxObserver(qscheme=scheme), HistogramObserver(qscheme=scheme)]\n",
+ " observer.forward(at)\n",
+ " o = observer.calculate_qparams()\n",
+ " # or observer(at)\n",
+ " #aq = _quantize_weight(at, observer), this method does not work well!\n",
+ " aq = torch.quantize_per_tensor(\n",
+ " torch.tensor(a),\n",
+ " o[0].item(), o[1].item(), torch.quint8)\n",
+ " print(a[0:10], aq[0:10])\n",
+ " print(np.linalg.norm((a-c)[0][0], 2), np.linalg.norm((a-cc), 2), torch.norm(at - stochastic_rounded, 2).item(), torch.norm(at - convert(aq), 2).item())\n",
+ " print(\"compression_ratio:\", len(b)/(len(a)*4), len(bb)/(len(a)*4), 0.25, 0.25)\n",
+ " return aq"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 102,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def flatten_model(state_dict):\n",
+ " shapes = []\n",
+ " lens = []\n",
+ " tensors_to_cat = []\n",
+ " for _, v in state_dict.items():\n",
+ " shapes.append(v.shape)\n",
+ " t = v.flatten()\n",
+ " lens.append(t.shape[0])\n",
+ " tensors_to_cat.append(t)\n",
+ " return torch.cat(tensors_to_cat)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 103,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "a = np.random.rand(100).astype(np.float32)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 104,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([0.14649276, 0.8268811 , 0.07076293, 0.41617852, 0.535295 ,\n",
+ " 0.09393147, 0.4459402 , 0.97720236, 0.58554167, 0.3981123 ],\n",
+ " dtype=float32)"
+ ]
+ },
+ "execution_count": 104,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "a[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 105,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[0.14649276 0.8268811 0.07076293 0.41617852 0.535295 0.09393147\n",
+ " 0.4459402 0.97720236 0.58554167 0.3981123 ] tensor([0.1465, 0.8287, 0.0694, 0.4163, 0.5357, 0.0925, 0.4471, 0.9790, 0.5858,\n",
+ " 0.3970], size=(10,), dtype=torch.quint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.0038542263209819794,\n",
+ " zero_point=0)\n",
+ "0.06702854 0.006093391 0.3343985676765442 0.011288280598819256\n",
+ "compression_ratio: 0.3525 0.42 0.25 0.25\n"
+ ]
+ }
+ ],
+ "source": [
+ "qt = comp(a)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 89,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def comp16(a):\n",
+ " b = fpzip.compress(a , precision=18, order='C')\n",
+ " c = fpzip.decompress(b, order='C')\n",
+ " \n",
+ " at = torch.from_numpy(a)\n",
+ " stochastic_rounded = float_quantize(at, exp=5, man=2, rounding=\"stochastic\")\n",
+ " \n",
+ " scheme = torch.per_tensor_affine#per_tensor_affine # affine means taking into account the actual range of the values\n",
+ " observer = MovingAverageMinMaxObserver(qscheme=scheme) #MinMaxObserver, MovingAverageMinMaxObserver(qscheme=scheme), HistogramObserver(qscheme=scheme)]\n",
+ " observer.forward(at)\n",
+ " o = observer.calculate_qparams()\n",
+ " # or observer(at)\n",
+ " #aq = _quantize_weight(at, observer), this method does not work well!\n",
+ " aq = torch.quantize_per_tensor(\n",
+ " torch.tensor(a),\n",
+ " o[0].item(), o[1].item(), torch.quint8)\n",
+ " print(a[0:10], aq[0:10])\n",
+ " print(np.linalg.norm((a-c)[0][0], 2), torch.norm(at - stochastic_rounded, 2).item(), torch.norm(at - convert(aq), 2).item())\n",
+ " print(\"compression_ratio:\", len(b)/(len(a)*4), 0.25, 0.25)\n",
+ " return aq"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 90,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "ValueError",
+ "evalue": "Data type float16 must be a floating type.",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
+ "Input \u001b[0;32mIn [90]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m qt \u001b[38;5;241m=\u001b[39m \u001b[43mcomp16\u001b[49m\u001b[43m(\u001b[49m\u001b[43ma\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mastype\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat16\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
+ "Input \u001b[0;32mIn [89]\u001b[0m, in \u001b[0;36mcomp16\u001b[0;34m(a)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcomp16\u001b[39m(a):\n\u001b[0;32m----> 2\u001b[0m b \u001b[38;5;241m=\u001b[39m \u001b[43mfpzip\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompress\u001b[49m\u001b[43m(\u001b[49m\u001b[43ma\u001b[49m\u001b[43m \u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprecision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m18\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mC\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3\u001b[0m c \u001b[38;5;241m=\u001b[39m fpzip\u001b[38;5;241m.\u001b[39mdecompress(b, order\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mC\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 5\u001b[0m at \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mfrom_numpy(a)\n",
+ "File \u001b[0;32mfpzip.pyx:91\u001b[0m, in \u001b[0;36mfpzip.compress\u001b[0;34m()\u001b[0m\n",
+ "\u001b[0;31mValueError\u001b[0m: Data type float16 must be a floating type."
+ ]
+ }
+ ],
+ "source": [
+ "qt = comp16(a.astype(np.float16))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 91,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAMfklEQVR4nO3cX4hc53nH8e9TSbFDLOqmGoyxvVm7hARTWlssbkqMoS5JbKnULfhCgSa5SFloY3CgpSgESnKnFhpKITRVGxPnr/PXNESkidsohEAjV3JkR7LqRkm3NMaNcIMT+yat3acX8+5qvJnRnv1zdp5ZfT8w7Jlz3j3nefWOfjrznnMUmYkkqa6fm3YBkqRLM6glqTiDWpKKM6glqTiDWpKK293HTvft25fz8/N97FqSdqRTp049m5mDcdt6Cer5+XlOnjzZx64laUeKiP+YtM2pD0kqzqCWpOIMakkqzqCWpOIMakkqzqCWpOI63Z4XEUvA88BLwIuZudBnUZKki9ZzH/VvZOazvVUiSRrLqQ9JKq7rGXUCX42IBP4mM4+ubhARi8AiwNzc3NZVWMj84WMry0tHDk6xEkmXk65n1Ldn5n7gbuBdEXHH6gaZeTQzFzJzYTAY+7i6JGkDOgV1Zj7dfl4AHgZu67MoSdJFawZ1RLwqIvYuLwNvBs70XZgkaajLHPU1wMMRsdz+k5n5D71WJUlasWZQZ+b3gV/dhlokSWN4e54kFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxnYM6InZFxLcj4kt9FiRJern1nFHfD5zrqxBJ0nidgjoirgcOAn/XbzmSpNV2d2z3l8CfAHsnNYiIRWARYG5ubtOFSRpv/vCxleWlIwenWIm2y5pn1BHxW8CFzDx1qXaZeTQzFzJzYTAYbFmBknS56zL18UbgtyNiCXgIuDMiPt5rVZKkFWsGdWa+JzOvz8x54BDwtcz8vd4rkyQB3kctSeV1vZgIQGZ+Hfh6L5VIksbyjFqSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJam4NYM6Iq6MiEcj4vGIOBsR79+OwiRJQ7s7tPkpcGdmvhARe4BvRsSXM/NbPdcmSaJDUGdmAi+0t3vaK/ssSpJ0Uac56ojYFRGngQvAI5l5oteqJEkrukx9kJkvAbdExNXAwxHxy5l5ZrRNRCwCiwBzc3NbXWcn84ePrSwvHTk49Rqq1DGtGkZV+XOZJdXGUNOzrrs+MvM54Dhw15htRzNzITMXBoPBFpUnSepy18egnUkTEa8E3gT8a891SZKaLlMf1wIPRsQuhsH+mcz8Ur9lSZKWdbnr4wng1m2oRZI0hk8mSlJxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxawZ1RNwQEccj4smIOBsR929HYZKkod0d2rwI/FFmPhYRe4FTEfFIZj7Zc22SJDqcUWfmM5n5WFt+HjgHXNd3YZKkoS5n1CsiYh64FTgxZtsisAgwNze3FbUxf/jYyvLSkYNj129mP5PajJrUvsvvbrTdRo/dpX1F6+1DtT5v5rOzlcfezGd1qz7nszh+s6DzxcSIuAr4PPDuzPzJ6u2ZeTQzFzJzYTAYbGWNknRZ6xTUEbGHYUh/IjO/0G9JkqRRXe76CODDwLnM/ED/JUmSRnU5o34j8Dbgzog43V4Heq5LktSseTExM78JxDbUIkkawycTJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSijOoJak4g1qSilszqCPigYi4EBFntqMgSdLLdTmj/ghwV891SJImWDOoM/MbwI+2oRZJ0hi7t2pHEbEILALMzc1t1W5XzB8+tuXt17vPjfxuH8dYOnJwy/cz2qaP/W9kX5Pq69K+y/5HbVWf19tms3Vs1bE38zmdtJ8u47eRfV2OtuxiYmYezcyFzFwYDAZbtVtJuux514ckFWdQS1JxXW7P+xTwz8DrIuIHEfHO/suSJC1b82JiZr51OwqRJI3n1IckFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFWdQS1JxBrUkFdcpqCPiroh4KiLOR8ThvouSJF20ZlBHxC7gg8DdwM3AWyPi5r4LkyQNdTmjvg04n5nfz8z/AR4C7um3LEnSssjMSzeIuBe4KzN/v71/G/BrmXnfqnaLwGJ7+zrgqQ3WtA94doO/W81O6ctO6QfYl4p2Sj9gc315TWYOxm3YvfF6Xi4zjwJHN7ufiDiZmQtbUNLU7ZS+7JR+gH2paKf0A/rrS5epj6eBG0beX9/WSZK2QZeg/hfgtRFxY0S8AjgEfLHfsiRJy9ac+sjMFyPiPuArwC7ggcw822NNm54+KWSn9GWn9APsS0U7pR/QU1/WvJgoSZoun0yUpOIMakkqrkxQz/pj6hGxFBHfiYjTEXGyrXt1RDwSEd9tP39h2nWOExEPRMSFiDgzsm5s7TH0V22cnoiI/dOr/GdN6Mv7IuLpNjanI+LAyLb3tL48FRFvmU7VPysiboiI4xHxZEScjYj72/qZG5dL9GUWx+XKiHg0Ih5vfXl/W39jRJxoNX+63XhBRFzR3p9v2+c3dODMnPqL4UXK7wE3Aa8AHgdunnZd6+zDErBv1bo/Bw635cPAn027zgm13wHsB86sVTtwAPgyEMAbgBPTrr9DX94H/PGYtje3z9oVwI3tM7hr2n1otV0L7G/Le4F/a/XO3Lhcoi+zOC4BXNWW9wAn2p/3Z4BDbf2HgD9oy38IfKgtHwI+vZHjVjmj3qmPqd8DPNiWHwR+Z3qlTJaZ3wB+tGr1pNrvAT6aQ98Cro6Ia7el0A4m9GWSe4CHMvOnmfnvwHmGn8Wpy8xnMvOxtvw8cA64jhkcl0v0ZZLK45KZ+UJ7u6e9ErgT+Fxbv3pclsfrc8BvRkSs97hVgvo64D9H3v+ASw9kRQl8NSJOtcfpAa7JzGfa8n8B10yntA2ZVPusjtV9bUrggZEpqJnoS/u6fCvDs7eZHpdVfYEZHJeI2BURp4ELwCMMz/ify8wXW5PRelf60rb/GPjF9R6zSlDvBLdn5n6G/8vguyLijtGNOfzuM5P3Qs5y7c1fA78E3AI8A/zFVKtZh4i4Cvg88O7M/MnotlkblzF9mclxycyXMvMWhk9p3wa8vu9jVgnqmX9MPTOfbj8vAA8zHMAfLn/9bD8vTK/CdZtU+8yNVWb+sP3l+j/gb7n4Nbp0XyJiD8Ng+0RmfqGtnslxGdeXWR2XZZn5HHAc+HWGU03LDxCO1rvSl7b954H/Xu+xqgT1TD+mHhGvioi9y8vAm4EzDPvwjtbsHcDfT6fCDZlU+xeBt7e7DN4A/Hjkq3hJq+Zqf5fh2MCwL4falfkbgdcCj253feO0ecwPA+cy8wMjm2ZuXCb1ZUbHZRARV7flVwJvYjjnfhy4tzVbPS7L43Uv8LX2TWh9pn0VdeRq6gGGV4O/B7x32vWss/abGF6lfhw4u1w/w7mofwK+C/wj8Opp1zqh/k8x/Or5vwzn1945qXaGV70/2MbpO8DCtOvv0JePtVqfaH9xrh1p/97Wl6eAu6dd/0hdtzOc1ngCON1eB2ZxXC7Rl1kcl18Bvt1qPgP8aVt/E8N/TM4DnwWuaOuvbO/Pt+03beS4PkIuScVVmfqQJE1gUEtScQa1JBVnUEtScQa1JBVnUEtScQa1JBX3/xTACMIG4l+aAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(qt.int_repr().to(torch.float32).numpy(), 100, (0, 300))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 121,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "a = (np.random.rand(10000).astype(np.float32) -0.5) *10"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 122,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([-4.3329406 , 2.0941095 , -0.43923616, 0.24154782, 3.528521 ,\n",
+ " -2.766264 , -0.47501475, 0.7699454 , 0.90143025, -4.2246346 ],\n",
+ " dtype=float32)"
+ ]
+ },
+ "execution_count": 122,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "a[0:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 123,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[-4.3329406 2.0941095 -0.43923616 0.24154782 3.528521 -2.766264\n",
+ " -0.47501475 0.7699454 0.90143025 -4.2246346 ] tensor([-4.3515, 2.0778, -0.4312, 0.2352, 3.5283, -2.7834, -0.4704, 0.7841,\n",
+ " 0.9017, -4.2339], size=(10,), dtype=torch.quint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.039202895015478134,\n",
+ " zero_point=127)\n",
+ "4.052173 0.016632441 23.100893020629883 1.1358001232147217\n",
+ "compression_ratio: 0.3867 0.5506 0.25 0.25\n"
+ ]
+ }
+ ],
+ "source": [
+ "qt = comp(a)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 95,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAARYElEQVR4nO3dfYxldX3H8fenPPpUAXdK1122u+i2hppWyQQxGmuktYC2SxNCIE1dKcmmLVatNgKaFPuHCfRBq9FgtkJdGspD0QZSbRURQ/oH2AV5ppQVQXazsGsV1Jqo6Ld/3LN4O8zszNwzT/c371cymXN+59x7vr85dz7zu7977p1UFZKktvzcchcgSVp4hrskNchwl6QGGe6S1CDDXZIadOhyFwCwZs2a2rhx43KXIUlj5Y477vhWVU1Mt21FhPvGjRvZuXPncpchSWMlyWMzbXNaRpIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgWcM9yRVJ9iW5b5pt701SSdZ060nysSS7ktyT5MTFKFqSdHBzeYfqp4GPA1cONyY5Dngz8M2h5tOAzd3Xa4DLuu/N2Hjh555dfvSStyxjJZI0s1lH7lV1K/DtaTZ9BHgfMPyvnLYAV9bAbcBRSdYuSKWSpDkbac49yRZgT1XdPWXTOuDxofXdXZskaQnN+4PDkjwfeD+DKZmRJdkGbAPYsGFDn7vSKuPUmDS7UUbuLwM2AXcneRRYD9yZ5BeBPcBxQ/uu79qeo6q2V9VkVU1OTEz7iZWSpBHNO9yr6t6q+oWq2lhVGxlMvZxYVU8ANwJv666aORl4uqr2LmzJkqTZzDotk+Rq4I3AmiS7gYur6vIZdv88cDqwC/gBcO4C1SnNaqVM1yxlHSulz1p5Zg33qjpnlu0bh5YLOL9/WRqFv+iSDlgR/4mpBQarpJXEjx+QpAY5ch/i6FtSKxy5S1KDDHdJapDhLkkNcs5dWga+vqPF5shdkhrkyH0Gjqy0HHzcaaEY7j0M/yL2ua2/xFoOPgbb5rSMJDXIkbu0zPo8A5RmYrjPwWr55fNputQOp2UkqUGO3KUR+UxHK5nhLi2R1TK9p5XBaRlJapAj90Xg0/Wfme9odbX/vFY6H9vjo+lwb+mB2FJfWuSUi1Yap2UkqUGGuyQ1aNZpmSRXAG8F9lXVK7u2vwZ+B/gR8HXg3Kp6qtt2EXAe8BPgnVX1hcUpvR19plycDpjeap/GmulxMcrPwsfYeJrLnPungY8DVw613QRcVFXPJLkUuAi4IMkJwNnArwIvBb6U5Jer6icLW/b0fBCqJav9D5T6mXVapqpuBb49pe2LVfVMt3obsL5b3gJcU1U/rKpvALuAkxawXknSHCzE1TJ/CFzbLa9jEPYH7O7aniPJNmAbwIYNGxagjNVjNT5DWY19Xo18trJweoV7kg8AzwBXzfe2VbUd2A4wOTlZferQ/KzmX6DV3HetLiOHe5K3M3ih9ZSqOhDOe4DjhnZb37VJkpbQSOGe5FTgfcBvVNUPhjbdCPxTkg8zeEF1M/DV3lUuIp/ujzfPnzS9uVwKeTXwRmBNkt3AxQyujjkCuCkJwG1V9UdVdX+S64AHGEzXnL9UV8poYTl9sXr1/YPpY2dlmDXcq+qcaZovP8j+HwI+1KcoSVI/TX+2zGrm6Ela3Qz3FWylzCfP5Q+Ff0ymt1LOYat83M3Mz5aRpAY5cl9k8x1ZLMZIb1xHj+Nat7QSGO7SLHzqr3FkuEud1RziS/EsaTX/fJeDc+6S1CBH7pJ66zPy97WVxeHIXZIaZLhLUoOclpG0aBZqusYXYOdvVYa7c3yri+d75XGOfvE5LSNJDVqVI3ctLkdW0vIz3Fc5g3j8rJRztpR1rJQ+jxOnZSSpQYa7JDXIaRmpEU5daJgjd0lqkCN3rVqOdNWyWcM9yRXAW4F9VfXKru0Y4FpgI/AocFZVfSdJgI8CpwM/AN5eVXcuTunz4y/ywluNP9PV2GeNp7lMy3waOHVK24XAzVW1Gbi5Wwc4DdjcfW0DLluYMiVJ8zHryL2qbk2ycUrzFuCN3fIO4CvABV37lVVVwG1Jjkqytqr2LljFY8xRn6SlMuqc+7FDgf0EcGy3vA54fGi/3V3bc8I9yTYGo3s2bNgwYhlqjX8ApYXR+2qZbpReI9xue1VNVtXkxMRE3zIkSUNGDfcnk6wF6L7v69r3AMcN7be+a5MkLaFRp2VuBLYCl3Tfbxhqf0eSa4DXAE87394Wp02k8TCXSyGvZvDi6Zoku4GLGYT6dUnOAx4Dzup2/zyDyyB3MbgU8txFqFnSiPzjvHrM5WqZc2bYdMo0+xZwft+iJEn9+PEDktQgP35AGgNOp2i+HLlLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUF+cJg0DT+oS+POkbskNchwl6QGGe6S1CDDXZIaZLhLUoN6hXuSP0tyf5L7klyd5Mgkm5LcnmRXkmuTHL5QxUqS5mbkcE+yDngnMFlVrwQOAc4GLgU+UlUvB74DnLcQhUqS5q7vtMyhwPOSHAo8H9gLvAm4vtu+Azij5zEkSfM0crhX1R7gb4BvMgj1p4E7gKeq6plut93Auulun2Rbkp1Jdu7fv3/UMiRJ0+gzLXM0sAXYBLwUeAFw6lxvX1Xbq2qyqiYnJiZGLUOSNI0+0zK/CXyjqvZX1Y+BzwKvA47qpmkA1gN7etYoSZqnPuH+TeDkJM9PEuAU4AHgFuDMbp+twA39SpQkzVefOffbGbxweidwb3df24ELgPck2QW8BLh8AeqUJM1Dr0+FrKqLgYunND8CnNTnfiVJ/fgOVUlqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgsf8H2f4jY0l6LkfuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDeoV7kmOSnJ9kv9K8mCS1yY5JslNSR7uvh+9UMVKkuam78j9o8C/V9UrgF8HHgQuBG6uqs3Azd26JGkJjRzuSV4MvAG4HKCqflRVTwFbgB3dbjuAM/qVKEmarz4j903AfuAfknwtyaeSvAA4tqr2dvs8ARw73Y2TbEuyM8nO/fv39yhDkjRVn3A/FDgRuKyqXg38L1OmYKqqgJruxlW1vaomq2pyYmKiRxmSpKn6hPtuYHdV3d6tX88g7J9Mshag+76vX4mSpPkaOdyr6gng8SS/0jWdAjwA3Ahs7dq2Ajf0qlCSNG99/0H2nwJXJTkceAQ4l8EfjOuSnAc8BpzV8xiSpHnqFe5VdRcwOc2mU/rcrySpH9+hKkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBvUO9ySHJPlakn/t1jcluT3JriTXJjm8f5mSpPlYiJH7u4AHh9YvBT5SVS8HvgOctwDHkCTNQ69wT7IeeAvwqW49wJuA67tddgBn9DmGJGn++o7c/w54H/DTbv0lwFNV9Uy3vhtYN90Nk2xLsjPJzv379/csQ5I0bORwT/JWYF9V3THK7atqe1VNVtXkxMTEqGVIkqZxaI/bvg743SSnA0cCPw98FDgqyaHd6H09sKd/mZKk+Rh55F5VF1XV+qraCJwNfLmqfh+4BTiz220rcEPvKiVJ87IY17lfALwnyS4Gc/CXL8IxJEkH0Wda5llV9RXgK93yI8BJC3G/kqTR+A5VSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KAFuRRSkpbbxgs/9+zyo5e8ZRkrWRkcuUtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDVo5HBPclySW5I8kOT+JO/q2o9JclOSh7vvRy9cuZKkuegzcn8GeG9VnQCcDJyf5ATgQuDmqtoM3NytS5KW0MjhXlV7q+rObvl7wIPAOmALsKPbbQdwRs8aJUnztCBz7kk2Aq8GbgeOraq93aYngGMX4hiSpLnrHe5JXgh8Bnh3VX13eFtVFVAz3G5bkp1Jdu7fv79vGZKkIb3CPclhDIL9qqr6bNf8ZJK13fa1wL7pbltV26tqsqomJyYm+pQhSZqiz9UyAS4HHqyqDw9tuhHY2i1vBW4YvTxJ0ij6/IPs1wF/ANyb5K6u7f3AJcB1Sc4DHgPO6lWhJGneRg73qvoPIDNsPmXU+5Uk9ec7VCWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIatGjhnuTUJA8l2ZXkwsU6jiTpuRYl3JMcAnwCOA04ATgnyQmLcSxJ0nMt1sj9JGBXVT1SVT8CrgG2LNKxJElTHLpI97sOeHxofTfwmuEdkmwDtnWr30/y0IjHWgN8a8TbrjT2ZWVqpS+t9ANm6UsuXcJK+utzXn5ppg2LFe6zqqrtwPa+95NkZ1VNLkBJy86+rEyt9KWVfoB9mYvFmpbZAxw3tL6+a5MkLYHFCvf/BDYn2ZTkcOBs4MZFOpYkaYpFmZapqmeSvAP4AnAIcEVV3b8Yx2IBpnZWEPuyMrXSl1b6AfZlVqmqxbhfSdIy8h2qktQgw12SGjTW4T7uH3GQ5NEk9ya5K8nOru2YJDclebj7fvRy1zmdJFck2ZfkvqG2aWvPwMe683RPkhOXr/L/b4Z+fDDJnu683JXk9KFtF3X9eCjJby9P1dNLclySW5I8kOT+JO/q2sfqvBykH2N3XpIcmeSrSe7u+vKXXfumJLd3NV/bXXhCkiO69V3d9o0jH7yqxvKLwQu1XweOBw4H7gZOWO665tmHR4E1U9r+CriwW74QuHS565yh9jcAJwL3zVY7cDrwb0CAk4Hbl7v+WfrxQeDPp9n3hO5xdgSwqXv8HbLcfRiqby1wYrf8IuC/u5rH6rwcpB9jd166n+0Lu+XDgNu7n/V1wNld+yeBP+6W/wT4ZLd8NnDtqMce55F7qx9xsAXY0S3vAM5YvlJmVlW3At+e0jxT7VuAK2vgNuCoJGuXpNBZzNCPmWwBrqmqH1bVN4BdDB6HK0JV7a2qO7vl7wEPMni3+Fidl4P0YyYr9rx0P9vvd6uHdV8FvAm4vmufek4OnKvrgVOSZJRjj3O4T/cRBwd7AKxEBXwxyR3dxzEAHFtVe7vlJ4Bjl6e0kcxU+zieq3d0UxVXDE2NjU0/uqfzr2YwUhzb8zKlHzCG5yXJIUnuAvYBNzF4ZvFUVT3T7TJc77N96bY/DbxklOOOc7i34PVVdSKDT888P8kbhjfW4LnZWF6rOs61A5cBLwNeBewF/nZZq5mnJC8EPgO8u6q+O7xtnM7LNP0Yy/NSVT+pqlcxeKf+ScArluK44xzuY/8RB1W1p/u+D/gXBif+yQNPjbvv+5avwnmbqfaxOldV9WT3C/lT4O/52VP8Fd+PJIcxCMSrquqzXfPYnZfp+jHO5wWgqp4CbgFey2AK7MCbSIfrfbYv3fYXA/8zyvHGOdzH+iMOkrwgyYsOLANvBu5j0Iet3W5bgRuWp8KRzFT7jcDbuqszTgaeHpomWHGmzDv/HoPzAoN+nN1d0bAJ2Ax8danrm0k3N3s58GBVfXho01idl5n6MY7nJclEkqO65ecBv8XgNYRbgDO73aaekwPn6kzgy92zrflb7leTe74SfTqDV9K/DnxgueuZZ+3HM3iF/27g/gP1M5hfuxl4GPgScMxy1zpD/VczeGr8YwZzhufNVDuDKwY+0Z2ne4HJ5a5/ln78Y1fnPd0v29qh/T/Q9eMh4LTlrn9KX17PYMrlHuCu7uv0cTsvB+nH2J0X4NeAr3U13wf8Rdd+PIM/QLuAfwaO6NqP7NZ3dduPH/XYfvyAJDVonKdlJEkzMNwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg/4PxKEPOlMHcVMAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(qt.int_repr().to(torch.float32).numpy(), 100, (0, 300))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 96,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[-0.17422378 -0.17071381 0.00554137 -0.02331001 0.03649271 0.08106142\n",
+ " 0.05222103 -0.13010341 -0.16420947 0.04394973] tensor([-0.1739, -0.1712, 0.0054, -0.0245, 0.0353, 0.0815, 0.0516, -0.1304,\n",
+ " -0.1630, 0.0435], size=(10,), dtype=torch.quint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.0027172896079719067,\n",
+ " zero_point=126)\n",
+ "0.1308374 0.1322985 0.7414445281028748 0.07875864207744598\n",
+ "compression_ratio: 0.3656 0.302 0.25 0.25\n"
+ ]
+ }
+ ],
+ "source": [
+ "a = np.random.normal(0, 0.1, 10000).astype(np.float32)\n",
+ "qt = comp(a)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 112,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ " bb = compress(a, tolerance=0.001, parallel=False)\n",
+ " cc = decompress(bb, a.shape, a.dtype, tolerance=0.001)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 119,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ " cc = decompress(np.array(bb).tobytes(), a.shape, a.dtype, tolerance=0.001)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 120,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([0.14685059, 0.8272705 , 0.07067871, 0.4161377 , 0.5352783 ,\n",
+ " 0.09411621, 0.44592285, 0.977417 , 0.58569336, 0.39819336,\n",
+ " 0.6730957 , 0.03930664, 0.45550537, 0.75909424, 0.93377686,\n",
+ " 0.59088135, 0.03643799, 0.9828491 , 0.35211182, 0.24383545,\n",
+ " 0.01055908, 0.00189209, 0.13970947, 0.24627686, 0.06787109,\n",
+ " 0.74121094, 0.5839844 , 0.87353516, 0.39404297, 0.55029297,\n",
+ " 0.5673828 , 0.28515625, 0.46887207, 0.90270996, 0.7393799 ,\n",
+ " 0.7327881 , 0.19091797, 0.87109375, 0.04248047, 0.5761719 ,\n",
+ " 0.42425537, 0.21954346, 0.9767456 , 0.04937744, 0.708313 ,\n",
+ " 0.05804443, 0.62750244, 0.51239014, 0.8782959 , 0.18518066,\n",
+ " 0.6590576 , 0.48742676, 0.12817383, 0.42944336, 0.7199707 ,\n",
+ " 0.92163086, 0.24468994, 0.8966675 , 0.3772583 , 0.7821655 ,\n",
+ " 0.9729614 , 0.96136475, 0.41363525, 0.5739136 , 0.7182617 ,\n",
+ " 0.5683594 , 0.12109375, 0.89990234, 0.29241943, 0.01593018,\n",
+ " 0.19989014, 0.2798462 , 0.49334717, 0.31988525, 0.22454834,\n",
+ " 0.80303955, 0.08361816, 0.9276123 , 0.29406738, 0.6204834 ,\n",
+ " 0.33605957, 0.7904053 , 0.25598145, 0.4515381 , 0.40979004,\n",
+ " 0.10681152, 0.3433838 , 0.36169434, 0.39416504, 0.25134277,\n",
+ " 0.6998291 , 0.32556152, 0.7110596 , 0.21276855, 0.98010254,\n",
+ " 0.9661865 , 0.04711914, 0.9538574 , 0.20532227, 0.7858887 ],\n",
+ " dtype=float32)"
+ ]
+ },
+ "execution_count": 120,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "cc"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 116,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "192"
+ ]
+ },
+ "execution_count": 116,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(np.array(bb).tobytes())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 118,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "192"
+ ]
+ },
+ "execution_count": 118,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(bb)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 117,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "400"
+ ]
+ },
+ "execution_count": 117,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(a.tobytes())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 49,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAASz0lEQVR4nO3dfYzlV33f8fcHY5uoPPhputnuLhkrbBpB1Cx0aohoG2pDY0zFOgpxjFJY0FabKkYiImlZkj/y0FoybcAJSmt1gylLlAZcJ8gr7DRxbCOEVBvGsHFYO9QTMPJuF+8EjAOycGXz7R9z1rnezOzcmXvvPJx5v6Sr+f3O79y532Pvfubsmd9DqgpJUl+et94FSJLGz3CXpA4Z7pLUIcNdkjpkuEtSh56/3gUAXHLJJTU9Pb3eZUjSpnL//ff/dVVNLXZsQ4T79PQ0s7Oz612GJG0qSb621DGXZSSpQ4a7JHXIcJekDhnuktQhw12SOjR0uCc5J8kXk3yq7V+a5L4kc0k+keS81n5+259rx6cnVLskaQkrmbm/G3hoYP/9wI1V9TLgcWB/a98PPN7ab2z9JElraKhwT7ITeBPw4bYf4HLg1tblMHB1297b9mnHr2j9JUlrZNiZ+28B/x74Xtu/GPhWVT3d9o8DO9r2DuBRgHb8idb/OZIcSDKbZHZ+fn511UuSFrXsFapJ/hVwqqruT/K6cX1wVR0CDgHMzMz4xBCtu+mDtz+7/cgNb1rHSqTRDXP7gdcCb05yFfAC4MXAbwMXJHl+m53vBE60/ieAXcDxJM8HXgJ8Y+yVS5KWtOyyTFW9r6p2VtU0cC1wd1X9LHAP8JbWbR9wW9s+0vZpx+8un+WnTWb64O3PvqTNaJTz3N8LvCfJHAtr6je39puBi1v7e4CDo5UoSVqpFd0Vsqo+DXy6bX8FuGyRPt8FfnoMtUmSVskrVCWpQ4a7JHVoQzysQ1oPq/llqadLarNw5i5JHTLcJalDLstIy/Bcd21Ghru65zq5tiLDXRoDf4Boo3HNXZI6ZLhLUodclpFWyV+0aiMz3LWlGMjaKlyWkaQOGe6S1CHDXZI6ZLhLUocMd0nq0LLhnuQFST6X5M+THEvy6639o0m+muRoe+1p7UnyoSRzSR5I8qoJj0GSdIZhToV8Cri8qr6T5Fzgs0n+uB37d1V16xn93wjsbq9XAze1r5KkNbLszL0WfKftnttedZa37AU+1t53L3BBku2jlypJGtZQa+5JzklyFDgF3FlV97VD17ellxuTnN/adgCPDrz9eGs783seSDKbZHZ+fn71I5Ca6YO3P/uStrqhwr2qnqmqPcBO4LIkPwK8D/hh4J8AFwHvXckHV9WhqpqpqpmpqamVVS1JOqsV3X6gqr6V5B7gyqr6zdb8VJL/DvxS2z8B7Bp4287WJm0J3v5XG8EwZ8tMJbmgbX8f8AbgL0+voycJcDXwpfaWI8Db21kzrwGeqKqTE6hdkrSEYWbu24HDSc5h4YfBLVX1qSR3J5kCAhwF/m3rfwdwFTAHPAm8c+xVS5LOatlwr6oHgFcu0n75Ev0LuG700iRJq+UVqpLUIe/nLk2Qv1zVenHmLkkdcuYurRFn8VpLztwlqUOGuyR1yHCXpA4Z7pLUIX+hqi55Z0htdc7cJalDhrskdchwl6QOueauTW2zrq17QZMmzZm7JHXIcJekDhnuktQhw12SOjTMM1RfkORzSf48ybEkv97aL01yX5K5JJ9Icl5rP7/tz7Xj0xMegyTpDMPM3J8CLq+qHwX2AFe2B1+/H7ixql4GPA7sb/33A4+39htbP0lLmD54+7MvaVyWDfda8J22e257FXA5cGtrPwxc3bb3tn3a8SuSZFwFS5KWN9Sae5JzkhwFTgF3An8FfKuqnm5djgM72vYO4FGAdvwJ4OJFvueBJLNJZufn50cahCTpuYYK96p6pqr2ADuBy4AfHvWDq+pQVc1U1czU1NSo306SNGBFZ8tU1beAe4AfAy5IcvoK153AibZ9AtgF0I6/BPjGOIqVJA1nmLNlppJc0La/D3gD8BALIf+W1m0fcFvbPtL2acfvrqoaY82SpGUMc2+Z7cDhJOew8MPglqr6VJIHgY8n+Y/AF4GbW/+bgd9LMgd8E7h2AnVLks5i2XCvqgeAVy7S/hUW1t/PbP8u8NNjqU6StCpeoSpJHTLcJalDhrskdciHdWjT8TJ9aXnO3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDnuUsbyOA5/I/c8KZ1rESbnTN3SeqQ4S5JHXJZRpuCtxyQVsaZuyR1yHCXpA4N8wzVXUnuSfJgkmNJ3t3afy3JiSRH2+uqgfe8L8lcki8n+YlJDkCS9HcNs+b+NPCLVfWFJC8C7k9yZzt2Y1X95mDnJC9n4bmprwD+AfBnSX6oqp4ZZ+GSpKUtO3OvqpNV9YW2/W3gIWDHWd6yF/h4VT1VVV8F5ljkWauSpMlZ0Zp7kmkWHpZ9X2t6V5IHknwkyYWtbQfw6MDbjrPID4MkB5LMJpmdn59feeWSpCUNHe5JXgj8IfALVfU3wE3ADwJ7gJPAB1bywVV1qKpmqmpmampqJW+VJC1jqHBPci4Lwf77VfVHAFX1WFU9U1XfA36Xv116OQHsGnj7ztYmSVojw5wtE+Bm4KGq+uBA+/aBbj8JfKltHwGuTXJ+kkuB3cDnxleyJGk5w5wt81rgbcBfJDna2n4ZeGuSPUABjwA/B1BVx5LcAjzIwpk213mmjCStrWXDvao+C2SRQ3ec5T3XA9ePUJckaQReoSpJHTLcJalDhrskdchwl6QOGe6S1CEf1qENywd0SKvnzF2SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjrkFarSBrXUFbqP3PCmNa5Em5Ezd0nq0DDPUN2V5J4kDyY5luTdrf2iJHcmebh9vbC1J8mHkswleSDJqyY9CEnScw2zLPM08ItV9YUkLwLuT3In8A7grqq6IclB4CDwXuCNLDwUezfwauCm9lXSGAwu17hEo6UsO3OvqpNV9YW2/W3gIWAHsBc43LodBq5u23uBj9WCe4ELkmwfd+GSpKWtaM09yTTwSuA+YFtVnWyHvg5sa9s7gEcH3na8tZ35vQ4kmU0yOz8/v9K6JUlnMfTZMkleCPwh8AtV9TdJnj1WVZWkVvLBVXUIOAQwMzOzoveqX97DXRqPoWbuSc5lIdh/v6r+qDU/dnq5pX091dpPALsG3r6ztUmS1sgwZ8sEuBl4qKo+OHDoCLCvbe8Dbhtof3s7a+Y1wBMDyzeSpDUwzLLMa4G3AX+R5Ghr+2XgBuCWJPuBrwHXtGN3AFcBc8CTwDvHWbD641KMNH7LhntVfRbIEoevWKR/AdeNWJckaQReoSpJHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDg39sA5JG4/PU9VSnLlLUoecuWtdeA93abKcuUtShwx3SerQMM9Q/UiSU0m+NND2a0lOJDnaXlcNHHtfkrkkX07yE5MqXJK0tGFm7h8Frlyk/caq2tNedwAkeTlwLfCK9p7/muSccRUrSRrOsuFeVZ8Bvjnk99sLfLyqnqqqr7LwkOzLRqhPkrQKo6y5vyvJA23Z5sLWtgN4dKDP8db2dyQ5kGQ2yez8/PwIZUiSzrTacL8J+EFgD3AS+MBKv0FVHaqqmaqamZqaWmUZkqTFrCrcq+qxqnqmqr4H/C5/u/RyAtg10HVna5MkraFVhXuS7QO7PwmcPpPmCHBtkvOTXArsBj43WomSpJVa9grVJH8AvA64JMlx4FeB1yXZAxTwCPBzAFV1LMktwIPA08B1VfXMRCqXJC1p2XCvqrcu0nzzWfpfD1w/SlGSpNF4haokdchwl6QOeVdIqRPe212DDHdNlIEjrQ+XZSSpQ4a7JHXIcJekDrnmrjXjo/WktePMXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDnmeu9Qh7+kjw11jYZhIG8uyyzJJPpLkVJIvDbRdlOTOJA+3rxe29iT5UJK5JA8kedUki5ckLW6YNfePAlee0XYQuKuqdgN3tX2AN7LwUOzdwAHgpvGUKUlaiWXDvao+A3zzjOa9wOG2fRi4eqD9Y7XgXuCCJNvHVKskaUirXXPfVlUn2/bXgW1tewfw6EC/463tJGdIcoCF2T0vfelLV1mGNiJvECatv5FPhayqAmoV7ztUVTNVNTM1NTVqGZKkAauduT+WZHtVnWzLLqda+wlg10C/na1N0jrxTKatabUz9yPAvra9D7htoP3t7ayZ1wBPDCzfSJLWyLIz9yR/ALwOuCTJceBXgRuAW5LsB74GXNO63wFcBcwBTwLvnEDNkqRlLBvuVfXWJQ5dsUjfAq4btShJ0mi8t4wkdcjbD0hblL9o7Zszd0nqkDN3aQvxArOtw5m7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yCtUtWpe7ShtXM7cJalDhrskdchwl6QOjbTmnuQR4NvAM8DTVTWT5CLgE8A08AhwTVU9PlqZkibJe7v3Zxwz939RVXuqaqbtHwTuqqrdwF1tX5K0hiaxLLMXONy2DwNXT+AzJElnMWq4F/CnSe5PcqC1bauqk23768C2xd6Y5ECS2SSz8/PzI5YhSRo06nnu/7SqTiT5+8CdSf5y8GBVVZJa7I1VdQg4BDAzM7NoH0nS6ow0c6+qE+3rKeCTwGXAY0m2A7Svp0YtUpK0MqueuSf5e8DzqurbbftfAr8BHAH2ATe0r7eNo1BtDF6VKm0OoyzLbAM+meT09/kfVfW/knweuCXJfuBrwDWjlylJWolVh3tVfQX40UXavwFcMUpRktbPmf8687z3zckbh2lZLsVIm4+3H5CkDhnuktQhl2W0KJdidJr3ndmcnLlLUoecuUsamrP4zcNw3+L8y6px8M/RxmO4S1oVfy+zsRnuepZ/WaV++AtVSeqQM/ctwMvJpa3HmbskdShV6/+cjJmZmZqdnV3vMrri+rk2Av+VOFlJ7h94fvVzOHOXpA655t4RZ+uSTjPcNyEvGNFmtNTkwz/Dk+Ga+ybkDF1bgaG/vLOtuU9s5p7kSuC3gXOAD1fVDZP6rF45Q9dW5p//0Uwk3JOcA/wX4A3AceDzSY5U1YOT+LyeOCuXNA6TmrlfBsy156yS5OPAXmDTh/u4ZhMr/T6GvjRevf/LYCJr7kneAlxZVf+m7b8NeHVVvWugzwHgQNv9h8CXx17I5F0C/PV6F7HGHHP/ttp4YfOO+QeqamqxA+t2tkxVHQIOrdfnj0OS2aV+mdErx9y/rTZe6HPMk7qI6QSwa2B/Z2uTJK2BSYX754HdSS5Nch5wLXBkQp8lSTrDRJZlqurpJO8C/oSFUyE/UlXHJvFZ62xTLyutkmPu31YbL3Q45g1xEZMkaby8cZgkdchwl6QOGe4rkOSiJHcmebh9vfAsfV+c5HiS31nLGsdtmDEn2ZPkfyc5luSBJD+zHrWOIsmVSb6cZC7JwUWOn5/kE+34fUmm16HMsRpizO9J8mD7f3pXkh9YjzrHabkxD/T7qSSVZNOeHmm4r8xB4K6q2g3c1faX8h+Az6xJVZM1zJifBN5eVa8ArgR+K8kFa1fiaAZul/FG4OXAW5O8/Ixu+4HHq+plwI3A+9e2yvEacsxfBGaq6h8BtwL/aW2rHK8hx0ySFwHvBu5b2wrHy3Bfmb3A4bZ9GLh6sU5J/jGwDfjTtSlropYdc1X9n6p6uG3/X+AUsOhVcxvUs7fLqKr/B5y+Xcagwf8OtwJXJMka1jhuy465qu6pqifb7r0sXK+ymQ3z/xkWJmbvB767lsWNm+G+Mtuq6mTb/joLAf4cSZ4HfAD4pbUsbIKWHfOgJJcB5wF/NenCxmgH8OjA/vHWtmifqnoaeAK4eE2qm4xhxjxoP/DHE61o8pYdc5JXAbuqatPfzMmHdZwhyZ8B37/IoV8Z3KmqSrLYeaQ/D9xRVcc3y8RuDGM+/X22A78H7Kuq7423Sq2XJP8amAF+fL1rmaQ2Mfsg8I51LmUsDPczVNXrlzqW5LEk26vqZAuyU4t0+zHgnyX5eeCFwHlJvlNVZ1ufX1djGDNJXgzcDvxKVd07oVInZZjbZZzuczzJ84GXAN9Ym/ImYqhbhCR5PQs/5H+8qp5ao9omZbkxvwj4EeDTbWL2/cCRJG+uqk33NCGXZVbmCLCvbe8DbjuzQ1X9bFW9tKqmWVia+dhGDvYhLDvmdouJT7Iw1lvXsLZxGeZ2GYP/Hd4C3F2b+wrAZcec5JXAfwPeXFWL/lDfZM465qp6oqouqarp9vf3XhbGvumCHQz3lboBeEOSh4HXt32SzCT58LpWNjnDjPka4J8D70hytL32rEu1q9DW0E/fLuMh4JaqOpbkN5K8uXW7Gbg4yRzwHs5+ptSGN+SY/zML//r8n+3/6aa+P9SQY+6Gtx+QpA45c5ekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUP/H+Wgr/3vyNjzAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(a, 100, (-0.5,0.5))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAASfklEQVR4nO3df4xdZ33n8fenIQ2ooIY0U8vrH+tAvULpauuksyFVUcsmoiTuHw4SmzV/QIoiudsmUpFoVdNKhZU2UlgVEJXYdM0mi1NRQhpAsdp0t2mIFPFHAk5qgpM0xYBRbJnYQBKIUNMmfPeP+zhczMzcmbkzc+c+fr+kqzn3Oefe+318xp957nPPOTdVhSSpLz816QIkSSvPcJekDhnuktQhw12SOmS4S1KHXjHpAgAuvPDC2rZt26TLkKSp8vDDD3+7qmbmWrcuwn3btm0cPHhw0mVI0lRJ8s351jktI0kdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHVoXZ6hKS7Ft79+8vHz05t+cYCXS+uXIXZI6ZLhLUocMd0nqkHPuWlecT5dWhiN3SeqQI3dNNUf60twcuUtShxy5q0uO6HW2M9w1FYbDWtJoTstIUocMd0nq0MhpmSSvBB4Azmvb31VV70/yCeDXgefapr9VVYeSBPgosBP4QWt/ZDWKl4YtZurGuXidLRYz5/4CcEVVPZ/kXOALSf62rfuDqrrrjO2vBra32xuBW9pPSdIaGTktUwPPt7vntlst8JBdwO3tcQ8C5yfZOH6pkqTFWtTRMknOAR4GfgH4WFU9lOR3gJuS/AlwH7C3ql4ANgFPDT38WGs7ccZz7gH2AGzdunXcfkjz8kgbnY0W9YFqVb1UVTuAzcBlSf498D7gDcB/BC4A/nApL1xV+6pqtqpmZ2Zmlla1JGlBSzpapqqeBe4HrqqqE23q5QXg/wCXtc2OA1uGHra5tUmS1sjIcE8yk+T8tvwq4C3AP56eR29Hx1wDHG4POQC8KwOXA89V1YmfeGJJ0qpZzJz7RmB/m3f/KeDOqvrrJJ9PMgMEOAT817b9PQwOgzzC4FDId6941ZKkBY0M96p6FLhkjvYr5tm+gBvGL02StFyeoSpJHfLCYVq3PIRRWj5H7pLUIcNdkjpkuEtShwx3SeqQ4S5JHfJoGZ21Fjoax2u9a9oZ7lozflGGtHYMd02EQS+tLufcJalDhrskdchwl6QOGe6S1CHDXZI65NEymjiv/iitPEfuktQhw12SOjRyWibJK4EHgPPa9ndV1fuTXATcAfwc8DDwzqr6lyTnAbcDvwx8B/gvVXV0leqXVsV8U0WecKVpsZiR+wvAFVX1S8AO4KoklwMfBD5SVb8APANc37a/HnimtX+kbSdJWkMjw70Gnm93z223Aq4A7mrt+4Fr2vKudp+2/sokWamCJUmjLWrOPck5SQ4BJ4F7ga8Bz1bVi22TY8CmtrwJeAqgrX+OwdTNmc+5J8nBJAdPnTo1VickST9uUeFeVS9V1Q5gM3AZ8IZxX7iq9lXVbFXNzszMjPt0kqQhSzrOvaqeTXI/8CvA+Ule0Ubnm4HjbbPjwBbgWJJXAD/L4INVaep5NUtNi5Ej9yQzSc5vy68C3gI8AdwPvL1tdh1wd1s+0O7T1n++qmoFa5YkjbCYkftGYH+Scxj8Mbizqv46yePAHUn+O/APwK1t+1uBv0hyBPgusHsV6pYkLWBkuFfVo8Alc7R/ncH8+5nt/wz85xWpTlPPSwtIk+EZqpLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CG/IFtaJi8ipvXMkbskdchwl6QOGe6S1CHDXZI65AeqWnFe5leaPEfuktQhw12SOmS4S1KHDHdJ6tDIcE+yJcn9SR5P8liS32vtH0hyPMmhdts59Jj3JTmS5Mkkb13NDkiSftJijpZ5EXhvVT2S5DXAw0nubes+UlV/OrxxkouB3cAvAv8G+Psk/66qXlrJwiVJ8xs5cq+qE1X1SFv+PvAEsGmBh+wC7qiqF6rqG8AR4LKVKFaStDhLmnNPsg24BHioNd2Y5NEktyV5bWvbBDw19LBjzPHHIMmeJAeTHDx16tTSK5ckzWvR4Z7k1cBngPdU1feAW4DXAzuAE8CHlvLCVbWvqmaranZmZmYpD5UkjbCocE9yLoNg/2RVfRagqp6uqpeq6ofAx/nR1MtxYMvQwze3NknSGlnM0TIBbgWeqKoPD7VvHNrsbcDhtnwA2J3kvCQXAduBL65cyZKkURZztMyvAu8EvpLkUGv7I+AdSXYABRwFfhugqh5LcifwOIMjbW7wSBlJWlsjw72qvgBkjlX3LPCYm4CbxqhLkjQGz1CVpA4Z7pLUIcNdkjpkuEtSh/wmJq0Iv31JWl8cuUtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ144TFphwxdRO3rzb06wEp3NRoZ7ki3A7cAGBt+Xuq+qPprkAuDTwDYG36F6bVU9075Q+6PATuAHwG9V1SOrU74myStBSuvXYqZlXgTeW1UXA5cDNyS5GNgL3FdV24H72n2Aq4Ht7bYHuGXFq5YkLWhkuFfVidMj76r6PvAEsAnYBexvm+0HrmnLu4Dba+BB4PwkG1e6cEnS/JY0555kG3AJ8BCwoapOtFXfYjBtA4Pgf2roYcda24mhNpLsYTCyZ+vWrUutW1pXnKLSerPoo2WSvBr4DPCeqvre8LqqKgbz8YtWVfuqaraqZmdmZpbyUEnSCIsK9yTnMgj2T1bVZ1vz06enW9rPk639OLBl6OGbW5skaY2MDPd29MutwBNV9eGhVQeA69rydcDdQ+3vysDlwHND0zeSpDWwmDn3XwXeCXwlyaHW9kfAzcCdSa4Hvglc29bdw+AwyCMMDoV890oWrMlybnlpPOZdkzIy3KvqC0DmWX3lHNsXcMOYdUmSxuDlBySpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pBf1qGRPHFJmj6O3CWpQ47cpTXipQi0lhy5S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIY9z15w8K1Wabov5guzbkpxMcnio7QNJjic51G47h9a9L8mRJE8meetqFS5Jmt9ipmU+AVw1R/tHqmpHu90DkORiYDfwi+0x/zPJOStVrCRpcUaGe1U9AHx3kc+3C7ijql6oqm8AR4DLxqhPkrQM48y535jkXcBB4L1V9QywCXhwaJtjrU3SEK8zo9W23KNlbgFeD+wATgAfWuoTJNmT5GCSg6dOnVpmGZKkuSwr3Kvq6ap6qap+CHycH029HAe2DG26ubXN9Rz7qmq2qmZnZmaWU4YkaR7LCvckG4fuvg04fSTNAWB3kvOSXARsB744XomSpKUaOeee5FPAm4ELkxwD3g+8OckOoICjwG8DVNVjSe4EHgdeBG6oqpdWpXJJ0rxGhntVvWOO5lsX2P4m4KZxipIkjcfLD0hShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOeT33s5zXOJH65MhdkjrkyF2aMN89aTU4cpekDhnuktQhw12SOmS4S1KHDHdJ6pBHy+hlw0dtSJpujtwlqUOGuyR1yHCXpA4Z7pLUoZHhnuS2JCeTHB5quyDJvUm+2n6+trUnyZ8lOZLk0SSXrmbxkqS5LWbk/gngqjPa9gL3VdV24L52H+BqYHu77QFuWZkyJUlLMTLcq+oB4LtnNO8C9rfl/cA1Q+2318CDwPlJNq5QrZKkRVrunPuGqjrRlr8FbGjLm4CnhrY71tp+QpI9SQ4mOXjq1KllliFJmsvYH6hWVQG1jMftq6rZqpqdmZkZtwxJ0pDlnqH6dJKNVXWiTbucbO3HgS1D221ubVpHPBNV6t9yw/0AcB1wc/t591D7jUnuAN4IPDc0faMJMcyls8/IcE/yKeDNwIVJjgHvZxDqdya5HvgmcG3b/B5gJ3AE+AHw7lWoWeqW38qklTIy3KvqHfOsunKObQu4YdyiJEnj8QxVSeqQl/yVpoDTNVoqR+6S1CFH7tI65VFOGocjd0nqkOEuSR1yWkaaMn64qsVw5C5JHTLcJalDTst0yiMtpLObI3dJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtSh8Y6iSnJUeD7wEvAi1U1m+QC4NPANuAocG1VPTNemZKkpViJM1T/U1V9e+j+XuC+qro5yd52/w9X4HUkncGLiGk+qzEtswvY35b3A9eswmtIkhYw7si9gL9LUsD/qqp9wIaqOtHWfwvYMNcDk+wB9gBs3bp1zDIEXk9G0o+MG+5vqqrjSX4euDfJPw6vrKpqwf8T2h+CfQCzs7NzbiNJWp6xpmWq6nj7eRL4HHAZ8HSSjQDt58lxi5QkLc2ywz3JzyR5zell4DeAw8AB4Lq22XXA3eMWKUlamnGmZTYAn0ty+nn+sqr+b5IvAXcmuR74JnDt+GVKkpZi2eFeVV8HfmmO9u8AV45TlCRpPJ6hKkkd8mv2ppAnrmgUf0fkyF2SOuTIfcp54pKkuThyl6QOGe6S1CGnZaROzDdFN9+Hq37o2jdH7pLUIUfu0lnED+DPHob7OrOYt9CSNIrhvo4Z6JKWyzl3SeqQ4S5JHXJaZh1w+kXSSjPcJ8RA1zTy2PjpYbivMkNc02Ch31NDfDo55y5JHXLkvkJ8u6qz2Xwjf/8vTI4jd0nq0KqN3JNcBXwUOAf431V182q91lpazAjdeXb1xN/n6bQq4Z7kHOBjwFuAY8CXkhyoqsdX4/Ukrb3FhL7TlZOzWiP3y4AjVfV1gCR3ALuAFQ/3lfzlWeoIxRGNtH74h+THrVa4bwKeGrp/DHjj8AZJ9gB72t3nkzy5zNe6EPg2QD64zGdYP17uSwfsy/oz0X6s8P/PBfsyZVkwzn75t/OtmNjRMlW1D9g37vMkOVhVsytQ0sTZl/Wpl7700g+wL4uxWkfLHAe2DN3f3NokSWtgtcL9S8D2JBcl+WlgN3BglV5LknSGVZmWqaoXk9wI/D8Gh0LeVlWPrcZrsQJTO+uIfVmfeulLL/0A+zJSqmo1nleSNEGeoSpJHTLcJalDUx3uSa5K8mSSI0n2TrqepUpyNMlXkhxKcrC1XZDk3iRfbT9fO+k655LktiQnkxweapuz9gz8WdtPjya5dHKV/7h5+vGBJMfbfjmUZOfQuve1fjyZ5K2TqXpuSbYkuT/J40keS/J7rX2q9ssC/Zi6/ZLklUm+mOTLrS//rbVflOShVvOn24EnJDmv3T/S1m9b9otX1VTeGHxQ+zXgdcBPA18GLp50XUvsw1HgwjPa/gewty3vBT446Trnqf3XgEuBw6NqB3YCfwsEuBx4aNL1j+jHB4Dfn2Pbi9vv2XnARe3375xJ92Govo3ApW35NcA/tZqnar8s0I+p2y/t3/bVbflc4KH2b30nsLu1/znwO235d4E/b8u7gU8v97WneeT+8iUOqupfgNOXOJh2u4D9bXk/cM3kSplfVT0AfPeM5vlq3wXcXgMPAucn2bgmhY4wTz/mswu4o6peqKpvAEcY/B6uC1V1oqoeacvfB55gcLb4VO2XBfoxn3W7X9q/7fPt7rntVsAVwF2t/cx9cnpf3QVcmSTLee1pDve5LnGw0C/AelTA3yV5uF2OAWBDVZ1oy98CNkymtGWZr/Zp3Fc3tqmK24amxqamH+3t/CUMRopTu1/O6AdM4X5Jck6SQ8BJ4F4G7yyeraoX2ybD9b7cl7b+OeDnlvO60xzuPXhTVV0KXA3ckOTXhlfW4L3ZVB6rOs21A7cArwd2ACeAD020miVK8mrgM8B7qup7w+umab/M0Y+p3C9V9VJV7WBwpv5lwBvW4nWnOdyn/hIHVXW8/TwJfI7Bjn/69Fvj9vPk5Cpcsvlqn6p9VVVPt/+QPwQ+zo/e4q/7fiQ5l0EgfrKqPtuap26/zNWPad4vAFX1LHA/8CsMpsBOn0Q6XO/LfWnrfxb4znJeb5rDfaovcZDkZ5K85vQy8BvAYQZ9uK5tdh1w92QqXJb5aj8AvKsdnXE58NzQNMG6c8a889sY7BcY9GN3O6LhImA78MW1rm8+bW72VuCJqvrw0Kqp2i/z9WMa90uSmSTnt+VXMfiOiycYhPzb22Zn7pPT++rtwOfbu62lm/SnyWN+Er2TwSfpXwP+eNL1LLH21zH4hP/LwGOn62cwv3Yf8FXg74ELJl3rPPV/isFb439lMGd4/Xy1Mzhi4GNtP30FmJ10/SP68Retzkfbf7aNQ9v/cevHk8DVk67/jL68icGUy6PAoXbbOW37ZYF+TN1+Af4D8A+t5sPAn7T21zH4A3QE+CvgvNb+ynb/SFv/uuW+tpcfkKQOTfO0jCRpHoa7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6tD/B+K9o4qpi8BQAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(qt.int_repr().to(torch.float32).numpy(), 100, (0, 300))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 97,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torchvision"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 98,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "r18 = torchvision.models.resnet18(True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 99,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat = flatten_model(r18.state_dict())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 111,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[-0.01041935 -0.00613561 -0.00180978 0.07484142 0.05661485 0.01708333\n",
+ " -0.01269388 0.01108271 0.00952757 -0.10992692] tensor([ 0.0000, 0.0000, 0.0000, 0.0652, 0.0652, 0.0000, 0.0000, 0.0000,\n",
+ " 0.0000, -0.1305], size=(10,), dtype=torch.quint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.06523405760526657,\n",
+ " zero_point=45)\n",
+ "1.3827991 0.5653769 7.915776252746582 52.67439651489258\n",
+ "compression_ratio: 0.3207514668609603 0.3097713573964291 0.25 0.25\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "tensor([0., 0., 0., ..., 0., 0., 0.], size=(11699132,), dtype=torch.quint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.06523405760526657,\n",
+ " zero_point=45)"
+ ]
+ },
+ "execution_count": 111,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "comp(flat.numpy())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 547,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def comp2(a):\n",
+ " b = fpzip.compress(a , precision=12, order='C')\n",
+ " c = fpzip.decompress(b, order='C')\n",
+ " \n",
+ " at = torch.from_numpy(a)\n",
+ " stochastic_rounded = float_quantize(at, exp=5, man=2, rounding=\"stochastic\")\n",
+ " \n",
+ " scheme = torch.per_tensor_symmetric#per_tensor_affine # affine means taking into account the actual range of the values\n",
+ " observer = HistogramObserver(qscheme=scheme) #MinMaxObserver, MovingAverageMinMaxObserver(qscheme=scheme), HistogramObserver(qscheme=scheme)]\n",
+ " observer.forward(at)\n",
+ " o = observer.calculate_qparams()\n",
+ " # or observer(at)\n",
+ " #aq = _quantize_weight(at, observer), this method does not work well!\n",
+ " aq = torch.quantize_per_tensor(\n",
+ " torch.tensor(a),\n",
+ " o[0].item(), o[1].item(), torch.quint8)\n",
+ " print(a[0:10], aq[0:10])\n",
+ " print(np.linalg.norm((a-c)[0][0], 2), torch.norm(at - stochastic_rounded, 2).item(), torch.norm(at - convert(aq), 2).item())\n",
+ " print(\"compression_ratio:\", len(b)/(len(a)*4), 0.25, 0.25)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 468,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[-0.01041935 -0.00613561 -0.00180978 0.07484142 0.05661485 0.01708333\n",
+ " -0.01269388 0.01108271 0.00952757 -0.10992692] tensor([-0.0130, 0.0000, 0.0000, 0.0780, 0.0520, 0.0130, -0.0130, 0.0130,\n",
+ " 0.0130, -0.1040], size=(10,), dtype=torch.quint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.013002387247979641,\n",
+ " zero_point=128)\n",
+ "5.376652 7.8146185874938965 28.545759201049805\n",
+ "compression_ratio: 0.2583094839856495 0.25 0.25\n"
+ ]
+ }
+ ],
+ "source": [
+ "comp2(flat.numpy())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 474,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "11699132"
+ ]
+ },
+ "execution_count": 474,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(flat)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 469,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(13.7179)"
+ ]
+ },
+ "execution_count": 469,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat.max()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 470,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(-2.9168)"
+ ]
+ },
+ "execution_count": 470,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flat.min()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 471,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# these are the reasons why it is not working"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 476,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "580"
+ ]
+ },
+ "execution_count": 476,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(flat[flat > 1])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Implement qint compression sceme where we ignore values > 1 by sending them seperately"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 556,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat2 = flat[(flat < 0.2)]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 557,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat3 = flat2[flat2 > -0.2]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 106,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'flat3' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
+ "Input \u001b[0;32mIn [106]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m qt \u001b[38;5;241m=\u001b[39m comp(\u001b[43mflat3\u001b[49m\u001b[38;5;241m.\u001b[39mnumpy())\n",
+ "\u001b[0;31mNameError\u001b[0m: name 'flat3' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "qt = comp(flat3.numpy())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 559,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAARF0lEQVR4nO3df6zddX3H8efL8sNlMJ32aggta3FlrvEn3jA2jTJ/bAUSumVOS3S6DW2yiXHxR1bjggz/AZeZaIayThvFTBCdc02oY25jYVHBXiYgLQGvUKUdsxUBZ8zEbu/9cb7Fw+XennPb03vu+fh8JDf3++PD+b4/93v74nM/3+/5nlQVkqTJ96RxFyBJGg0DXZIaYaBLUiMMdElqhIEuSY0w0CWpEWMN9CTbkuxPcueQ7V+TZHeSXUk+dazrk6RJknHeh57kpcAPgKur6jkD2q4DrgNeXlUPJXlGVe1fijolaRKMdYReVTcB3+vfluRZSf4xya1J/j3Js7tdbwaurKqHuv/WMJekPstxDn0r8NaqehHwTuDD3fYzgDOSfCnJzUk2jK1CSVqGjht3Af2SnAT8GvCZJIc2n9h9Pw5YB5wDrAJuSvLcqnp4icuUpGVpWQU6vb8YHq6qF8yzby9wS1X9GLgvyT30An7nEtYnScvWsppyqarv0wvr3wVIz/O73Z+nNzonyUp6UzD3jqFMSVqWxn3b4jXAV4BfSrI3yUXA64CLktwO7AI2ds1vAB5Mshu4EXhXVT04jrolaTka622LkqTRWVZTLpKkIze2i6IrV66sNWvWjOvwkjSRbr311u9W1dR8+8YW6GvWrGFmZmZch5ekiZTkWwvtc8pFkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIasdyehy4NtGbL9Y8t77n8/DFWIi0vBromQn+IS5qfUy6S1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjBgZ6km1J9ie5c4H9r0tyR5KvJ/lykuePvkxJ0iDDjNA/Dmw4zP77gJdV1XOB9wFbR1CXJGmRBj5tsapuSrLmMPu/3Ld6M7BqBHVJkhZp1HPoFwFfWGhnks1JZpLMHDhwYMSHlqSfbiML9CS/Ti/Q/3ShNlW1taqmq2p6ampqVIeWJDGiD7hI8jzgo8C5VfXgKF5TkrQ4Rz1CT3Ia8Dng96rqnqMvSZJ0JAaO0JNcA5wDrEyyF3gvcDxAVV0FXAI8HfhwEoCDVTV9rAqWJM1vmLtcLhyw/03Am0ZWkbQIfmC09BO+U1SSGjGSi6LSsdA/+pY0mCN0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYMDPQk25LsT3LnAvuT5ENJZpPckeTM0ZcpSRpkmBH6x4ENh9l/LrCu+9oMfOToy5IkLdbAQK+qm4DvHabJRuDq6rkZeGqSU0ZVoCRpOKOYQz8VuL9vfW+37QmSbE4yk2TmwIEDIzi0JOmQJb0oWlVbq2q6qqanpqaW8tCS1LxRBPo+YHXf+qpumyRpCY0i0LcDb+judjkbeKSqHhjB60qSFuG4QQ2SXAOcA6xMshd4L3A8QFVdBewAzgNmgR8Cf3CsipUkLWxgoFfVhQP2F/CWkVUkSToiAwNdmhRrtlz/2PKey88fYyXSePjWf0lqhIEuSY0w0CWpEc6ha1npnweXtDiO0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNWKoQE+yIcndSWaTbJln/2lJbkzytSR3JDlv9KVKkg5nYKAnWQFcCZwLrAcuTLJ+TrM/A66rqhcCm4APj7pQSdLhDTNCPwuYrap7q+pR4Fpg45w2Bfxct/wU4D9HV6IkaRjDBPqpwP1963u7bf0uBV6fZC+wA3jrfC+UZHOSmSQzBw4cOIJyJUkLGdVF0QuBj1fVKuA84JNJnvDaVbW1qqaranpqampEh5YkwXCBvg9Y3be+qtvW7yLgOoCq+grwZGDlKAqUJA1nmEDfCaxLsjbJCfQuem6f0+bbwCsAkvwyvUB3TkWSltDAQK+qg8DFwA3AXfTuZtmV5LIkF3TN3gG8OcntwDXA71dVHauiJUlPdNwwjapqB72Lnf3bLulb3g28eLSlSZIWw3eKSlIjDHRJaoSBLkmNGGoOXZo0a7Zc/9jynsvPH2Ml0tJxhC5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIa4W2LGrv+WwwlHTlH6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiOGCvQkG5LcnWQ2yZYF2rwmye4ku5J8arRlSpIGGfhwriQrgCuBVwF7gZ1JtlfV7r4264B3Ay+uqoeSPONYFSxJmt8wI/SzgNmqureqHgWuBTbOafNm4MqqegigqvaPtkxJ0iDDBPqpwP1963u7bf3OAM5I8qUkNyfZMKoCJUnDGdXz0I8D1gHnAKuAm5I8t6oe7m+UZDOwGeC0004b0aElSTDcCH0fsLpvfVW3rd9eYHtV/biq7gPuoRfwj1NVW6tquqqmp6amjrRmSdI8hgn0ncC6JGuTnABsArbPafN5eqNzkqykNwVz7+jKlCQNMjDQq+ogcDFwA3AXcF1V7UpyWZILumY3AA8m2Q3cCLyrqh48VkVLkp5oqDn0qtoB7Jiz7ZK+5QLe3n1JksbAd4pKUiMMdElqhIEuSY0w0CWpEQa6JDViVO8UlZatNVuuf2x5z+Xnj7ES6dhyhC5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiO8D11j0X9vuKTRcIQuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNGCrQk2xIcneS2SRbDtPud5JUkunRlShJGsbAQE+yArgSOBdYD1yYZP087U4G3gbcMuoiJUmDDTNCPwuYrap7q+pR4Fpg4zzt3gdcAfzPCOuTJA1pmEA/Fbi/b31vt+0xSc4EVlfVYZ+4lGRzkpkkMwcOHFh0sZKkhR31RdEkTwI+ALxjUNuq2lpV01U1PTU1dbSHliT1GSbQ9wGr+9ZXddsOORl4DvBvSfYAZwPbvTAqSUtrmEDfCaxLsjbJCcAmYPuhnVX1SFWtrKo1VbUGuBm4oKpmjknFkqR5DfyAi6o6mORi4AZgBbCtqnYluQyYqarth38FafmY+8Eaey4/f0yVSKM31CcWVdUOYMecbZcs0Pacoy9LkrRYvlNUkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNWKo2xalUZh7D7ik0XKELkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRvvVfP9X6H0fgx9Fp0jlCl6RGGOiS1AgDXZIaYaBLUiMMdElqxFB3uSTZAHwQWAF8tKoun7P/7cCbgIPAAeAPq+pbI65VE8gPtZCWzsARepIVwJXAucB64MIk6+c0+xowXVXPAz4LvH/UhUqSDm+YKZezgNmqureqHgWuBTb2N6iqG6vqh93qzcCq0ZYpSRpkmEA/Fbi/b31vt20hFwFfmG9Hks1JZpLMHDhwYPgqJUkDjfSiaJLXA9PAX8y3v6q2VtV0VU1PTU2N8tCS9FNvmIui+4DVfeurum2Pk+SVwHuAl1XVj0ZTniRpWMOM0HcC65KsTXICsAnY3t8gyQuBvwYuqKr9oy9TkjTIwBF6VR1McjFwA73bFrdV1a4klwEzVbWd3hTLScBnkgB8u6ouOIZ1SyPng7o06Ya6D72qdgA75my7pG/5lSOuS5K0SL5TVJIaYaBLUiP8gAuNnG/3l8bDEbokNcJAl6RGGOiS1AgDXZIa4UVRaR6+yUiTyBG6JDXCQJekRjjlopHw3nNp/ByhS1IjDHRJaoRTLtIA3vGiSeEIXZIa4QhdR8wLodLy4ghdkhrhCF1aBOfTtZwZ6FoUp1mk5ctAl46Qo3UtNwa6BnJULk0GA13zMsQXx9G6lgMDXY8xxEfDcNe4DBXoSTYAHwRWAB+tqsvn7D8RuBp4EfAg8Nqq2jPaUjUqBvfSWehnbdDrWBgY6ElWAFcCrwL2AjuTbK+q3X3NLgIeqqpfTLIJuAJ47bEoWAszqCfHMOfK0NdiDTNCPwuYrap7AZJcC2wE+gN9I3Bpt/xZ4K+SpKpqhLU2xfDVIMv9d8T/4Sw/wwT6qcD9fet7gV9ZqE1VHUzyCPB04Lv9jZJsBjZ3qz9IcveRFA2snPvaE8y+LE+t9OWY9SNXHItXPaxWzgkcXV9+YaEdS3pRtKq2AluP9nWSzFTV9AhKGjv7sjy10pdW+gH2ZRjDPMtlH7C6b31Vt23eNkmOA55C7+KoJGmJDBPoO4F1SdYmOQHYBGyf02Y78MZu+dXAvzp/LklLa+CUSzcnfjFwA73bFrdV1a4klwEzVbUd+BjwySSzwPfohf6xdNTTNsuIfVmeWulLK/0A+zJQHEhLUht8HrokNcJAl6RGTFygJ9mQ5O4ks0m2jLuexUqyJ8nXk9yWZKbb9rQkX0zyje77z4+7zvkk2ZZkf5I7+7bNW3t6PtSdpzuSnDm+yh9vgX5cmmRfd15uS3Je3753d/24O8lvjqfq+SVZneTGJLuT7Erytm77RJ2Xw/Rj4s5Lkicn+WqS27u+/Hm3fW2SW7qaP93dZEKSE7v12W7/miM+eFVNzBe9i7LfBE4HTgBuB9aPu65F9mEPsHLOtvcDW7rlLcAV465zgdpfCpwJ3DmoduA84AtAgLOBW8Zd/4B+XAq8c56267vfsxOBtd3v34px96GvvlOAM7vlk4F7upon6rwcph8Td166n+1J3fLxwC3dz/o6YFO3/Srgj7rlPwau6pY3AZ8+0mNP2gj9sccQVNWjwKHHEEy6jcAnuuVPAL81vlIWVlU30buLqd9CtW8Erq6em4GnJjllSQodYIF+LGQjcG1V/aiq7gNm6f0eLgtV9UBV/Ue3/N/AXfTeuT1R5+Uw/VjIsj0v3c/2B93q8d1XAS+n92gUeOI5OXSuPgu8IkmO5NiTFujzPYbgcCd9OSrgn5Lc2j0KAeCZVfVAt/xfwDPHU9oRWaj2STxXF3fTENv6pr0mph/dn+ovpDcinNjzMqcfMIHnJcmKJLcB+4Ev0vsL4uGqOtg16a/3cY9OAQ49OmXRJi3QW/CSqjoTOBd4S5KX9u+s3t9dE3kv6STXDnwEeBbwAuAB4C/HWs0iJTkJ+DvgT6rq+/37Jum8zNOPiTwvVfW/VfUCeu+sPwt49lIcd9ICfZjHECxrVbWv+74f+Ht6J/s7h/7s7b7vH1+Fi7ZQ7RN1rqrqO90/wv8D/oaf/Pm+7PuR5Hh6Ifi3VfW5bvPEnZf5+jHJ5wWgqh4GbgR+ld701qE3c/bXO7JHp0xaoA/zGIJlK8nPJjn50DLwG8CdPP7RCW8E/mE8FR6RhWrfDryhu6vibOCRvimAZWfOPPJv0zsv0OvHpu5OhLXAOuCrS13fQrq51o8Bd1XVB/p2TdR5Wagfk3hekkwleWq3/DP0PkviLnrB/uqu2dxzMppHp4z7ivARXEE+j94V8G8C7xl3PYus/XR6V+ZvB3Ydqp/efNm/AN8A/hl42rhrXaD+a+j92ftjenOAFy1UO70r/Vd25+nrwPS46x/Qj092dd7R/QM7pa/9e7p+3A2cO+765/TlJfSmU+4Abuu+zpu083KYfkzceQGeB3ytq/lO4JJu++n0/qczC3wGOLHb/uRufbbbf/qRHtu3/ktSIyZtykWStAADXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXi/wFPibVeiXsUAgAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(qt.int_repr().to(torch.float32).numpy(), 100, (0, 300))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 548,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[-0.01041935 -0.00613561 -0.00180978 0.07484142 0.05661485 0.01708333\n",
+ " -0.01269388 0.01108271 0.00952757 -0.10992692] tensor([-0.0112, -0.0056, 0.0000, 0.0730, 0.0561, 0.0168, -0.0112, 0.0112,\n",
+ " 0.0112, -0.1123], size=(10,), dtype=torch.quint8,\n",
+ " quantization_scheme=torch.per_tensor_affine, scale=0.005614420399069786,\n",
+ " zero_point=128)\n",
+ "4.759155 6.922706604003906 5.910922527313232\n",
+ "compression_ratio: 0.2583142617722315 0.25 0.25\n"
+ ]
+ }
+ ],
+ "source": [
+ "comp2(flat3.numpy())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# need non linear quantization"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pywt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# removing bn layers"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "resw = r18.state_dict()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "resw = {k:v for k,v in resw.items() if \"bn1.\" not in k}\n",
+ "resw = {k:v for k,v in resw.items() if \"bn2.\" not in k}\n",
+ "resw = {k:v for k,v in resw.items() if len(v.shape) >1}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(102.5824)"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Random\n",
+ "probs = torch.rand_like(flat)\n",
+ "indices = probs < 0.1\n",
+ "top10_og = torch.zeros(len(flat))\n",
+ "top10_og[indices] = flat[indices]\n",
+ "torch.norm(top10_og - flat, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 102.74525451660156 185069.125\n",
+ "db1 103.23194122314453 185122.640625\n",
+ "sym2 103.66918182373047 184979.96875\n",
+ "coif1 102.05806732177734 184744.890625\n",
+ "bior1.1 102.97911834716797 185177.625\n",
+ "rbio1.1 103.00313568115234 185171.53125\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Random\n",
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "# wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = 4)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " probs = torch.rand_like(torch.from_numpy(array))\n",
+ " indices = probs < 0.1\n",
+ " \n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[indices] = torch.from_numpy(array[indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(25.6738)"
+ ]
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " flat.abs(), round(0.33*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ "top10_og = torch.zeros(len(flat))\n",
+ "top10_og[topk_og.indices] = flat[topk_og.indices]\n",
+ "torch.norm(top10_og - flat, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "torch.Size([11699132])\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(flat.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 20.19877052307129 53562.796875\n",
+ "db1 20.19877052307129 53562.796875\n",
+ "sym2 20.11495018005371 53570.296875\n",
+ "min: tensor(20.1150) sym2 2\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Base case on flattened data, with level=4\n",
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2']#, 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "# wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = 4)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.33*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0.9491062664912424"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# RMSE\n",
+ "import math\n",
+ "wv = math.sqrt(42.6776)\n",
+ "tpk = math.sqrt(47.3773)\n",
+ "wv / tpk\n",
+ "# --> 0.949 --> 5% less error"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0.8851460621415522"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "wv = math.sqrt(20.1150)\n",
+ "tpk = math.sqrt(25.6738)\n",
+ "wv / tpk\n",
+ "# --> 0.88 --> 12% less error\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# level 4: 42.677608489990234 114424.265625\n",
+ "# None: 42.69436264038086 114436.171875"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 59,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "approx = coeff[0]\n",
+ "details = coeff[1:]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 60,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([-0.02112804, -0.12861666, 0.355115 , ..., 0.01942001,\n",
+ " 0.00414812, -0.02489999], dtype=float32)"
+ ]
+ },
+ "execution_count": 60,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "approx"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "731195.75"
+ ]
+ },
+ "execution_count": 61,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(flat) / 16"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 62,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "731198"
+ ]
+ },
+ "execution_count": 62,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(approx)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 63,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "16.845043 -5.0946984 -0.00447843\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(approx.max(), approx.min(), approx.mean())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 64,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAASHUlEQVR4nO3df6zddX3H8ed7VNDplAI3HbaNF2O3BRNFclcwOp2itMJm+QMdbuqN6dK41czFJbPMJWw4MtwfY5ooWyNkxWwDxuZowMFqwWxLBvSiWC0Ee0FIWwu90oJuTBz63h/nc/VYz+05955zz6/P85Hc3O/38/2ccz7ve3tf38/5nO85jcxEklSHnxn0ACRJ/WPoS1JFDH1JqoihL0kVMfQlqSIrBj2AEznjjDNycnJy0MOQpJFy//33fzszJ1odG+rQn5ycZGZmZtDDkKSREhGPL3TM5R1JqoihL0kVMfQlqSKGviRVxNCXpIoY+pJUEUNfkipi6EtSRQx9SaqIoS9JFTH0Jakihr4kVcTQlxZpctvtgx6CtGSGvrQIBr5GXUehHxGPRcTXIuKBiJgpbadFxK6I2F++ryztERGfiojZiNgbEec23c906b8/IqaXpySptwx6jZPFzPTfkpnnZOZU2d8G7M7MdcDusg/wDmBd+doCXAuNkwRwBXAesB64Yv5EIY0Cw1/joJvlnU3AjrK9A7ikqf2GbLgHODUizgQ2ALsy82hmHgN2ARu7eHxJ0iJ1GvoJ/FtE3B8RW0rbqsw8XLafAFaV7dXAgabbHixtC7X/hIjYEhEzETEzNzfX4fAkSZ3oNPTfmJnn0li62RoRb2o+mJlJ48TQtczcnplTmTk1MdHyv3iU+uJEyzku9WhUdRT6mXmofD8CfJ7GmvyTZdmG8v1I6X4IWNt08zWlbaF2SVKftA39iHhxRPzc/DZwIfB1YCcwfwXONHBr2d4JvL9cxXM+8ExZBroTuDAiVpYXcC8sbdJIcravUbSigz6rgM9HxHz/v8/MOyJiD3BzRGwGHgfeXfp/AbgImAWeBT4AkJlHI+LjwJ7S78rMPNqzSqRlZMBrXLQN/cx8FHhti/angAtatCewdYH7uh64fvHDlCT1gu/IlU6g3QzfZwAaNYa+JFXE0Jekihj6klQRQ1+SKmLoS1JFDH2pBa/K0bgy9KXjLDbwPUFolBj6UhMDXOPO0Jekihj6klQRQ1/qAZeFNCoMfUmqiKEvSRUx9KWi2yUal3g0Cgx9SaqIoS9JFTH0Jakihr4kVcTQl6SKGPqSVBFDX6J3l1t62aaGnaEvSRUx9CWpIoa+JFXE0Jekihj6klQRQ1+SKmLoS1JFDH1Jqoihr6r5ZirVpuPQj4iTIuIrEXFb2T8rIu6NiNmIuCkiTi7tp5T92XJ8suk+Li/tD0fEhp5XIy2Bwa+aLGam/2Hgoab9TwDXZOargGPA5tK+GThW2q8p/YiIs4HLgFcDG4HPRMRJ3Q1fGj6T2273RKKh1VHoR8Qa4GLgs2U/gLcCt5QuO4BLyvamsk85fkHpvwm4MTOfy8xvArPA+h7UIEnqUKcz/b8C/hD4Ydk/HXg6M58v+weB1WV7NXAAoBx/pvT/UXuL2/xIRGyJiJmImJmbm+u8EmnIONvXMGob+hHxa8CRzLy/D+MhM7dn5lRmTk1MTPTjISWpGis66PMG4J0RcRHwQuClwCeBUyNiRZnNrwEOlf6HgLXAwYhYAbwMeKqpfV7zbSRJfdB2pp+Zl2fmmsycpPFC7F2Z+VvA3cClpds0cGvZ3ln2Kcfvysws7ZeVq3vOAtYB9/WsEklSW53M9BfyUeDGiPgz4CvAdaX9OuBzETELHKVxoiAz90XEzcCDwPPA1sz8QRePL0lapEWFfmZ+CfhS2X6UFlffZOb3gHctcPurgKsWO0hJUm/4jlxJqoihL0kVMfQlqSKGvqrlm6dUI0Nfkipi6EvLyGcTGjaGviRVxNCXpIoY+qqSyy6qlaEvSRUx9CWpIoa+JFXE0JeWma8faJgY+pJUEUNf1XHmrZoZ+pJUEUNfkipi6EtSRQx9SaqIoS9JFTH0VRWv3FHtDH2pDzzZaFgY+pJUEUNfkipi6EtSRQx9SaqIoS9JFTH0pT7xCh4NA0Nf1TB0JUNfkqrSNvQj4oURcV9EfDUi9kXEn5b2syLi3oiYjYibIuLk0n5K2Z8txyeb7uvy0v5wRGxYtqokSS11MtN/DnhrZr4WOAfYGBHnA58ArsnMVwHHgM2l/2bgWGm/pvQjIs4GLgNeDWwEPhMRJ/WwFklSG21DPxv+u+y+oHwl8FbgltK+A7ikbG8q+5TjF0RElPYbM/O5zPwmMAus70URkqTOdLSmHxEnRcQDwBFgF/AI8HRmPl+6HARWl+3VwAGAcvwZ4PTm9ha3aX6sLRExExEzc3Nziy5IkrSwjkI/M3+QmecAa2jMzn9puQaUmdszcyozpyYmJpbrYVQZr9yRGhZ19U5mPg3cDbweODUiVpRDa4BDZfsQsBagHH8Z8FRze4vbSJL6oJOrdyYi4tSy/SLg7cBDNML/0tJtGri1bO8s+5Tjd2VmlvbLytU9ZwHrgPt6VIc0EnzGoUHrZKZ/JnB3ROwF9gC7MvM24KPARyJilsaa/XWl/3XA6aX9I8A2gMzcB9wMPAjcAWzNzB/0shhpFBj8GqQV7Tpk5l7gdS3aH6XF1TeZ+T3gXQvc11XAVYsfpiSpF3xHriRVxNCXpIoY+pJUEUNfkipi6EtSRQx9SaqIoa+x53Xx0o8Z+pJUEUNfGgCffWhQDH1JqoihL0kVMfQ11lxGkX6SoS9JFTH0Jakihr4kVcTQl6SKGPqSVBFDXxoQryzSIBj6klQRQ19jy5m09NMMfUmqiKEvSRUx9CWpIoa+NEC+7qB+M/Q1lgxTqTVDX5IqYuhLUkUMfUmqiKEvSRUx9KUB80Vn9ZOhr7FjiEoLaxv6EbE2Iu6OiAcjYl9EfLi0nxYRuyJif/m+srRHRHwqImYjYm9EnNt0X9Ol//6ImF6+siRJrXQy038e+IPMPBs4H9gaEWcD24DdmbkO2F32Ad4BrCtfW4BroXGSAK4AzgPWA1fMnyik2vnsRP3SNvQz83Bmfrlsfxd4CFgNbAJ2lG47gEvK9ibghmy4Bzg1Is4ENgC7MvNoZh4DdgEbe1mMJOnEFrWmHxGTwOuAe4FVmXm4HHoCWFW2VwMHmm52sLQt1H78Y2yJiJmImJmbm1vM8CRJbXQc+hHxEuCfgN/PzO80H8vMBLIXA8rM7Zk5lZlTExMTvbhLSVLRUehHxAtoBP7fZeY/l+Yny7IN5fuR0n4IWNt08zWlbaF2SVKfdHL1TgDXAQ9l5l82HdoJzF+BMw3c2tT+/nIVz/nAM2UZ6E7gwohYWV7AvbC0SZL6ZEUHfd4AvA/4WkQ8UNr+CLgauDkiNgOPA+8ux74AXATMAs8CHwDIzKMR8XFgT+l3ZWYe7UUR0jyvgpFOLBrL8cNpamoqZ2ZmBj0MjZBRD/3Hrr540EPQGIiI+zNzqtUx35ErSRUx9CWpIoa+JFXE0Jekihj60hAZ9ReiNfwMfY0NA1Nqz9CXpIoY+pJUEUNfY8GlHakzhr4kVcTQl4aMz1q0nAx9SaqIoS9JFTH0Jakihr5GnmvgUucMfUmqiKEvDSGfvWi5GPqSVBFDX5IqYuhLUkUMfUmqiKGvkTbOL3iOc20aHENfkipi6Gtk1TATrqFG9ZehL0kVMfQlqSKGviRVxNCXhpzr+uolQ1+SKmLoayQ5+5WWxtCXpIq0Df2IuD4ijkTE15vaTouIXRGxv3xfWdojIj4VEbMRsTcizm26zXTpvz8ippenHEnSiXQy0/9bYONxbduA3Zm5Dthd9gHeAawrX1uAa6FxkgCuAM4D1gNXzJ8opMVyaUdaurahn5n/Dhw9rnkTsKNs7wAuaWq/IRvuAU6NiDOBDcCuzDyamceAXfz0iUTSAjzRqVeWuqa/KjMPl+0ngFVlezVwoKnfwdK2UPtPiYgtETETETNzc3NLHJ7GleEndafrF3IzM4HswVjm7297Zk5l5tTExESv7lYaeZ7w1AtLDf0ny7IN5fuR0n4IWNvUb01pW6hdktRHSw39ncD8FTjTwK1N7e8vV/GcDzxTloHuBC6MiJXlBdwLS5vUMWe6/gzUvRXtOkTEPwC/CpwREQdpXIVzNXBzRGwGHgfeXbp/AbgImAWeBT4AkJlHI+LjwJ7S78rMPP7FYUnSMmsb+pn5ngUOXdCibwJbF7if64HrFzU6SVJP+Y5cSaqIoS+NGNf11Q1DX5IqYuhrJDi7/Un+PLRUhr6GngEn9Y6hL40oT4ZaCkNfkipi6GuoOZuVesvQl0aYJ0UtlqGvoWWgdcafkxbD0Jekihj6klQRQ19DySULaXm0/ZRNqZ8Me2l5OdOXxsDktts9Yaojhr4kVcTQl8aIs321Y+hLY8bg14kY+pJUEUNfQ8MZau/4s9RCDH0NBUOq9/yZqhVDXwNnOC0fL+XU8Qx9DZSBJPWXoa+BMfD7x5+15hn66juXHAbDn7nA0FefGTyD5c9ffuCa+sbAGQ7Nv4fHrr54gCPRIDjT17KaDxgDfzj5e6lPZOagx7CgqampnJmZGfQwtEQGyuhx5j8eIuL+zJxqdczlHfWUQT/a5n9/j119MZPbbvckMIac6asnDPvx5wlgdAzVTD8iNgKfBE4CPpuZV/d7DOrO/AzQoK9Lq9+3J4LR09eZfkScBHwDeDtwENgDvCczH2zV35l+/zUHusGubh1/UnDJqD9ONNPvd+i/HviTzNxQ9i8HyMw/b9Xf0P+x4/9Yjr/sznBWTZr/zR//+kOrv5WFjo2rYQr9S4GNmfnbZf99wHmZ+aGmPluALWX3F4GHu3jIM4Bvd3H7YTEudYC1DKNxqQOsZd4rMnOi1YGhu3onM7cD23txXxExs9DZbpSMSx1gLcNoXOoAa+lEv9+cdQhY27S/prRJkvqg36G/B1gXEWdFxMnAZcDOPo9BkqrV1+WdzHw+Ij4E3Enjks3rM3PfMj5kT5aJhsC41AHWMozGpQ6wlraG+s1ZkqTe8gPXJKkihr4kVWSsQj8iTouIXRGxv3xf2aLPKyLiyxHxQETsi4gPDmKsJ9JhHedExH+VGvZGxG8MYqztdFJL6XdHRDwdEbf1e4wnEhEbI+LhiJiNiG0tjp8SETeV4/dGxOQAhtmRDmp5U/nbeL68p2ZodVDLRyLiwfK3sTsiXjGIcXaig1o+GBFfK5n1nxFxdlcPmJlj8wX8BbCtbG8DPtGiz8nAKWX7JcBjwMsHPfYl1PELwLqy/XLgMHDqoMe+lFrKsQuAXwduG/SYm8Z0EvAI8Mry7+arwNnH9fld4K/L9mXATYMedxe1TAKvAW4ALh30mLus5S3Az5bt3xnx38tLm7bfCdzRzWOO1Uwf2ATsKNs7gEuO75CZ38/M58ruKQzns51O6vhGZu4v298CjgAt34E3YG1rAcjM3cB3+zSmTq0HZjPz0cz8PnAjjXqaNdd3C3BBREQfx9iptrVk5mOZuRf44SAGuAid1HJ3Zj5bdu+h8Z6gYdRJLd9p2n0x0NXVN8MYeN1YlZmHy/YTwKpWnSJibUTsBQ7QmHl+q18D7FBHdcyLiPU0ZgmPLPfAlmBRtQyZ1TT+jcw7WNpa9snM54FngNP7MrrF6aSWUbHYWjYD/7qsI1q6jmqJiK0R8QiNZ86/180DDt3HMLQTEV8Efr7FoY8172RmRkTLM2JmHgBeExEvB/4lIm7JzCd7P9qF9aKOcj9nAp8DpjNzIDO0XtUi9VpEvBeYAt486LF0IzM/DXw6In4T+GNgeqn3NXKhn5lvW+hYRDwZEWdm5uEShkfa3Ne3IuLrwK/QeGreN72oIyJeCtwOfCwz71mmobbVy9/JkOnkY0Pm+xyMiBXAy4Cn+jO8RRmnj0DpqJaIeBuNicebm5Z0h81ify83Atd284Djtryzkx+fAaeBW4/vEBFrIuJFZXsl8Ea6+yTP5dBJHScDnwduyMy+nrAWqW0tQ6yTjw1pru9S4K4sr7gNmXH6CJS2tUTE64C/Ad6ZmcM80eiklnVNuxcD+7t6xEG/et3jV8JPB3aXH8oXgdNK+xSN/6ULGv+By14ar5LvBbYMetxLrOO9wP8BDzR9nTPosS+llrL/H8Ac8L801jU3DHrsZVwX0fiPfx6h8YwK4EoaYQLwQuAfgVngPuCVgx5zF7X8cvnZ/w+NZyv7Bj3mLmr5IvBk09/GzkGPuYtaPgnsK3XcDby6m8fzYxgkqSLjtrwjSToBQ1+SKmLoS1JFDH1JqoihL0kVMfQlqSKGviRV5P8BG17N1nxNq/MAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(approx, 1000, (-0.3, 0.3))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 65,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "4"
+ ]
+ },
+ "execution_count": 65,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(details)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 66,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[731198, 1462394, 2924785, 5849567]\n"
+ ]
+ }
+ ],
+ "source": [
+ "print([len(d) for d in details])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 67,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "d0 = details[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 68,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "4.3342423 -3.6128068 -5.1483516e-06\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(d0.max(), d0.min(), d0.mean())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 69,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([-0.02881567, -0.49651563, -0.06628633, ..., 0.02382721,\n",
+ " -0.02910749, 0.01628537], dtype=float32)"
+ ]
+ },
+ "execution_count": 69,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "d0"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 70,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAATaUlEQVR4nO3df5BdZ33f8fcnVmxSUpAMimokDXImChkzE4xna5tJmkwQyMK0yH841PmFhlFHk8ZJ02lnWlE649aEKfSPUpghbhSsVGaSGNctsQYojhDOJJmJjdfBEdgO0eLASMK2Nsh20tBATb794z4bLmJX96727r1397xfMzv3nOc899znu6v9nHOfc+4qVYUkqRu+a9IDkCSNj6EvSR1i6EtShxj6ktQhhr4kdciGSQ/gQl7+8pfXjh07Jj0MSVpTHnnkkb+oqs2LbZvq0N+xYwezs7OTHoYkrSlJvrzUNqd3JKlDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfeki7Dj48UkPQboohr4kdYihL0kdMlToJ9mY5N4kf5rkiSSvS3J5kmNJTrbHTa1vknwgyVySE0mu6dvPvtb/ZJJ9q1WUNC5O82itGfZM//3AJ6vqh4DXAE8AB4HjVbUTON7WAd4E7GxfB4A7AJJcDtwGXAdcC9y2cKCQ1gpDXmvdwNBP8lLgx4A7AarqG1X1HLAXONK6HQFuast7gbuq50FgY5IrgBuAY1V1rqqeBY4Be0ZYizQWBr/WsmHO9K8E5oHfSPLZJB9K8mJgS1U91fo8DWxpy1uBU33PP93almqX1oSlwt6DgNaSYUJ/A3ANcEdVvRb4a741lQNAVRVQoxhQkgNJZpPMzs/Pj2KXkqRmmNA/DZyuqofa+r30DgLPtGkb2uPZtv0MsL3v+dta21Lt36aqDlXVTFXNbN686P/2JUm6SANDv6qeBk4leVVr2gU8DhwFFu7A2Qfc15aPAm9rd/FcDzzfpoHuB3Yn2dQu4O5ubdKa5LSO1qJh/4/cXwJ+M8mlwJPA2+kdMO5Jsh/4MvDW1vcTwI3AHPC11peqOpfkXcDDrd/tVXVuJFVIkoYyVOhX1aPAzCKbdi3St4Bbl9jPYeDwMsYnSRohP5ErDcGpHK0Xhr4kdYihL0kdYuhLUocY+tIAw8znO+evtcLQl0bE4NdaYOhLUocY+pLUIYa+JHWIoS9dgPP0Wm8MfUnqEENfkjrE0JekDjH0pRHzOoCmmaEvSR1i6EtShxj60gg5taNpZ+hLUocY+pLUIYa+JHWIoS9JHWLoS1KHGPrSErwTR+uRoS9JHWLoS1KHDBX6Sb6U5HNJHk0y29ouT3Isycn2uKm1J8kHkswlOZHkmr797Gv9TybZtzolSZKWspwz/Z+oqquraqatHwSOV9VO4HhbB3gTsLN9HQDugN5BArgNuA64Frht4UAhrTdeD9C0Wsn0zl7gSFs+AtzU135X9TwIbExyBXADcKyqzlXVs8AxYM8KXl9aNYa21qthQ7+A303ySJIDrW1LVT3Vlp8GtrTlrcCpvueebm1LtX+bJAeSzCaZnZ+fH3J4kqRhbBiy349W1Zkk3wccS/Kn/RurqpLUKAZUVYeAQwAzMzMj2ackqWeoM/2qOtMezwIfpTcn/0ybtqE9nm3dzwDb+56+rbUt1S5JGpOBoZ/kxUn+/sIysBv4PHAUWLgDZx9wX1s+Cryt3cVzPfB8mwa6H9idZFO7gLu7tUnrktcFNI2Gmd7ZAnw0yUL/36qqTyZ5GLgnyX7gy8BbW/9PADcCc8DXgLcDVNW5JO8CHm79bq+qcyOrRJI00MDQr6ongdcs0v5VYNci7QXcusS+DgOHlz9MSdIo+Ilc6TxOy2g9M/QlqUMMfUnqEENfkjrE0JekDjH0JalDDH2pj3fuaL0z9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfWkXeAqppY+hLUocY+pLUIYa+1DgVoy4w9CWpQwx9SeoQQ1+SOsTQl1aZ1wo0TQx9SeoQQ1+SOsTQl6QOGTr0k1yS5LNJPtbWr0zyUJK5JB9Jcmlrv6ytz7XtO/r28Y7W/oUkN4y8GukiOe+urljOmf4vA0/0rb8XeF9V/QDwLLC/te8Hnm3t72v9SHIVcAvwamAP8KtJLlnZ8KW1wYOKpsVQoZ9kG/Bm4ENtPcDrgXtblyPATW15b1unbd/V+u8F7q6qr1fVnwNzwLUjqEGSNKRhz/T/K/BvgL9t6y8DnquqF9r6aWBrW94KnAJo259v/f+ufZHn/J0kB5LMJpmdn58fvhJJ0kADQz/JPwbOVtUjYxgPVXWoqmaqambz5s3jeElJ6owNQ/T5EeAtSW4EXgS8BHg/sDHJhnY2vw040/qfAbYDp5NsAF4KfLWvfUH/cyRJYzDwTL+q3lFV26pqB70LsZ+uqp8BHgBubt32Afe15aNtnbb901VVrf2WdnfPlcBO4DMjq0SSNNAwZ/pL+bfA3Ul+BfgscGdrvxP4cJI54By9AwVV9ViSe4DHgReAW6vqmyt4fUnSMi0r9Kvq94Dfa8tPssjdN1X1N8BPLvH8dwPvXu4gJUmj4SdypTHxXn1NA0NfnWcYq0sMfUnqEENfkjrE0JekDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9dZofzFLXGPrSGHmQ0aQZ+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj66iw/KKUuMvSlMfNgo0ky9CWpQwx9aQI829ekGPqS1CEDQz/Ji5J8JsmfJHksyX9s7VcmeSjJXJKPJLm0tV/W1ufa9h19+3pHa/9CkhtWrSpJ0qKGOdP/OvD6qnoNcDWwJ8n1wHuB91XVDwDPAvtb//3As639fa0fSa4CbgFeDewBfjXJJSOsRZI0wMDQr57/01a/u30V8Hrg3tZ+BLipLe9t67Ttu5Kktd9dVV+vqj8H5oBrR1GEJGk4Q83pJ7kkyaPAWeAY8EXguap6oXU5DWxty1uBUwBt+/PAy/rbF3lO/2sdSDKbZHZ+fn7ZBUmSljZU6FfVN6vqamAbvbPzH1qtAVXVoaqaqaqZzZs3r9bLSFInLevunap6DngAeB2wMcmGtmkbcKYtnwG2A7TtLwW+2t++yHOksfKWSXXVMHfvbE6ysS1/D/BG4Al64X9z67YPuK8tH23rtO2frqpq7be0u3uuBHYCnxlRHZKkIWwY3IUrgCPtTpvvAu6pqo8leRy4O8mvAJ8F7mz97wQ+nGQOOEfvjh2q6rEk9wCPAy8At1bVN0dbjiTpQgaGflWdAF67SPuTLHL3TVX9DfCTS+zr3cC7lz9MSdIo+IlcdY7z+eoyQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrE0JekDjH01SnerqmuM/SlCfEApEkw9CWpQwx9SeoQQ1+aIKd4NG6GviR1iKEvSR1i6EtShxj6ktQhhr46Y1ovmk7ruLQ+GfqS1CGGviR1iKEvSR1i6EtShxj60hTwYq7GxdCXpA4x9CWpQwaGfpLtSR5I8niSx5L8cmu/PMmxJCfb46bWniQfSDKX5ESSa/r2ta/1P5lk3+qVJX07p0+knmHO9F8A/nVVXQVcD9ya5CrgIHC8qnYCx9s6wJuAne3rAHAH9A4SwG3AdcC1wG0LBwpJ0ngMDP2qeqqq/rgt/xXwBLAV2Ascad2OADe15b3AXdXzILAxyRXADcCxqjpXVc8Cx4A9oyxGWst8N6JxWNacfpIdwGuBh4AtVfVU2/Q0sKUtbwVO9T3tdGtbqv381ziQZDbJ7Pz8/HKGJ0kaYOjQT/K9wP8E/mVV/WX/tqoqoEYxoKo6VFUzVTWzefPmUexSHecZtPQtQ4V+ku+mF/i/WVX/qzU/06ZtaI9nW/sZYHvf07e1tqXaJUljMszdOwHuBJ6oqv/St+kosHAHzj7gvr72t7W7eK4Hnm/TQPcDu5Nsahdwd7c2SdKYbBiiz48APwd8Lsmjre3fAe8B7kmyH/gy8Na27RPAjcAc8DXg7QBVdS7Ju4CHW7/bq+rcKIqQJA1nYOhX1R8CWWLzrkX6F3DrEvs6DBxezgAlSaPjJ3IlqUMMfa1r3rkjfTtDX5I6xNCXpojvTLTaDH1J6hBDX5oynu1rNRn6ktQhhr4kdYihr3XLaRLpOxn6ktQhhr7WJc/ypcUZ+tIU8qCl1WLoS1KHGPqS1CGGvtYdp0akpRn6ktQhhr40xXzXolEz9KUpZeBrNRj6ktQhhr4kdYihL0kdYuhLUocY+lpX1uPFz/VYkybH0JekDhkY+kkOJzmb5PN9bZcnOZbkZHvc1NqT5ANJ5pKcSHJN33P2tf4nk+xbnXIkSRcyzJn+fwf2nNd2EDheVTuB420d4E3AzvZ1ALgDegcJ4DbgOuBa4LaFA4UkaXwGhn5V/T5w7rzmvcCRtnwEuKmv/a7qeRDYmOQK4AbgWFWdq6pngWN854FEWhHnvqXBLnZOf0tVPdWWnwa2tOWtwKm+fqdb21Lt3yHJgSSzSWbn5+cvcnjqGgNfGs6KL+RWVQE1grEs7O9QVc1U1czmzZtHtVtpTfOgplG52NB/pk3b0B7PtvYzwPa+ftta21LtkqQxutjQPwos3IGzD7ivr/1t7S6e64Hn2zTQ/cDuJJvaBdzdrU2SNEbD3LL528AfAa9KcjrJfuA9wBuTnATe0NYBPgE8CcwBvw78AkBVnQPeBTzcvm5vbdKKdWXqoyt1anVtGNShqn5qiU27FulbwK1L7OcwcHhZo5MkjZSfyJWkDjH0pTXEKR6tlKEvrTEGv1bC0NeaZgBKy2Poa80y8KXlM/QlqUMMfWkN8l2OLpahL61RBr8uhqGvNcnA6/H7oOUy9CWpQwb+GQZpmnhmK62MZ/qS1CGGvtYMz/IX5/dFy2HoS+uAwa9hGfpaEwy1wfweaRiGvqaeYSaNjqGvqbXj4McN/GXy+6VBDH1pnfFgqQsx9DWVDK2V83uoxfjhLE0Vg2q0Fr6fX3rPmyc8Ek0Lz/QlqUMMfU0Fz/BXn3P9AkhVTXoMS5qZmanZ2dlJD0OryBCaDKd71rckj1TVzGLbnNPX2Bn0k9c/17/j4Mc9CHSIZ/paVQb82mL4rw9TdaafZA/wfuAS4ENV9Z5xj0Gry6Bfu5b62XkwWD/GGvpJLgE+CLwROA08nORoVT0+znFoeOe/9TfQu2mxn/tiU0NOFU2/sU7vJHkd8B+q6oa2/g6AqvpPi/V3emf5zr8ve+GX0LDWWnGhf6/LPaB09SB0oemdcYf+zcCeqvpnbf3ngOuq6hf7+hwADrTVVwFfWMFLvhz4ixU8f1qslzrAWqbReqkDrGXBK6tq82Ibpu7unao6BBwaxb6SzC51tFtL1ksdYC3TaL3UAdYyjHF/OOsMsL1vfVtrkySNwbhD/2FgZ5Irk1wK3AIcHfMYJKmzxjq9U1UvJPlF4H56t2werqrHVvElRzJNNAXWSx1gLdNovdQB1jLQVH84S5I0Wv7BNUnqEENfkjpkXYV+ksuTHEtysj1uWqTPK5P8cZJHkzyW5OcnMdYLGbKOq5P8UavhRJJ/OomxDjJMLa3fJ5M8l+Rj4x7jhSTZk+QLSeaSHFxk+2VJPtK2P5RkxwSGOZQhavmx9rvxQvtMzdQaopZ/leTx9rtxPMkrJzHOYQxRy88n+VzLrD9MctWKXrCq1s0X8J+Bg235IPDeRfpcClzWlr8X+BLwikmP/SLq+EFgZ1t+BfAUsHHSY7+YWtq2XcA/AT426TH3jekS4IvA97d/N38CXHVen18A/ltbvgX4yKTHvYJadgA/DNwF3DzpMa+wlp8A/l5b/udr/Ofykr7ltwCfXMlrrqszfWAvcKQtHwFuOr9DVX2jqr7eVi9jOt/tDFPHn1XVybb8FeAssOgn8CZsYC0AVXUc+KsxjWlY1wJzVfVkVX0DuJtePf3667sX2JUkYxzjsAbWUlVfqqoTwN9OYoDLMEwtD1TV19rqg/Q+EzSNhqnlL/tWXwys6O6baQy8ldhSVU+15aeBLYt1SrI9yQngFL0zz6+Ma4BDGqqOBUmupXeW8MXVHthFWFYtU2YrvX8jC063tkX7VNULwPPAy8YyuuUZppa1Yrm17Af+96qO6OINVUuSW5N8kd4753+xkhecuj/DMEiSTwH/YJFN7+xfqapKsugRsapOAT+c5BXA7yS5t6qeGf1olzaKOtp+rgA+DOyrqomcoY2qFmnUkvwsMAP8+KTHshJV9UHgg0l+Gvj3wL6L3deaC/2qesNS25I8k+SKqnqqheHZAfv6SpLPA/+I3lvzsRlFHUleAnwceGdVPbhKQx1olD+TKTPMnw1Z6HM6yQbgpcBXxzO8ZVlPfwJlqFqSvIHeiceP903pTpvl/lzuBu5YyQuut+mdo3zrCLgPuO/8Dkm2JfmetrwJ+FFW9pc8V8MwdVwKfBS4q6rGesBapoG1TLFh/mxIf303A5+udsVtyqynP4EysJYkrwV+DXhLVU3zicYwtezsW30zcHJFrzjpq9cjvhL+MuB4+6Z8Cri8tc/Q+1+6oPcfuJygd5X8BHBg0uO+yDp+Fvh/wKN9X1dPeuwXU0tb/wNgHvi/9OY1b5j02Nu4bgT+jN71kne2ttvphQnAi4D/AcwBnwG+f9JjXkEt/7B97/+a3ruVxyY95hXU8ingmb7fjaOTHvMKank/8Fir4wHg1St5Pf8MgyR1yHqb3pEkXYChL0kdYuhLUocY+pLUIYa+JHWIoS9JHWLoS1KH/H/ucjoIogs+HgAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(d0, 1000, (-0.3, 0.3))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 78,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 83.8722915649414 171346.921875\n",
+ "db1 83.8722915649414 171346.921875\n",
+ "sym2 83.61727142333984 169984.34375\n",
+ "min: tensor(83.6173) sym2 2\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Just using the approximation!\n",
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2']#, 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "# wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = 4)\n",
+ " approx = coeff[0]\n",
+ " details = coeff[1:]\n",
+ " details = [np.zeros_like(d) for d in details]\n",
+ " approx_only = [approx]\n",
+ " approx_only.extend(details)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(approx_only)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " #topk = torch.topk(\n",
+ " # torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " # )\n",
+ " #top10 = torch.zeros(len(array))\n",
+ " #top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(array, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "18.828228"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "array[topk.indices].max()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "-9.629911"
+ ]
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "array[topk.indices].min()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sub = array[topk.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "527"
+ ]
+ },
+ "execution_count": 34,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(sub[np.absolute(sub) > 1])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1169917"
+ ]
+ },
+ "execution_count": 35,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(sub)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAATcklEQVR4nO3db7Bc9X3f8ffHUsBuUowwqkIkxleeqHFJOsX4DlbiTlODLQTuWHSKXXmaoriq1cQ4k0ySSUT9gNYOU9wHIWbqOKVGRbitZUrqQQ1QVebPZDpjYS41BguKdcH2IAUjxQIcj8c44G8f7O96jsXee/dKe3evpPdrZmfP+Z7fOfvdc1f72T17dpWqQpJ0envNuBuQJI2fYSBJMgwkSYaBJAnDQJIELB93A8fr3HPPrYmJiXG3IUknjYcffvgvq2plv2UnbRhMTEwwNTU17jYk6aSR5JuzLfMwkSTJMJAkGQaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQGDIMk30jyWJJHkky12jlJ9iY50K5XtHqS3JRkOsmjSS7qbGdLG38gyZZO/a1t+9Nt3Qz7jkqSZreQdwbvqKoLq2qyzW8H7q2qdcC9bR7gcmBdu2wDPgW98ACuA94GXAxcNxMgbcwHO+ttPO57JElasBM5TLQJ2NmmdwJXduq3Vc8+4Owk5wGXAXur6mhVPQ/sBTa2ZWdV1b6qKuC2zrYkSSMwaBgU8L+TPJxkW6utqqpn2/S3gFVtejXwTGfdg602V/1gn/qrJNmWZCrJ1JEjRwZsXZI0n0H/P4O/X1WHkvwtYG+S/9ddWFWVpIbf3o+rqpuBmwEmJycX/fYk6XQx0DuDqjrUrg8Dn6d3zP+5doiHdn24DT8EnN9ZfU2rzVVf06cuSRqRecMgyU8m+Zsz08AG4KvAbmDmjKAtwJ1tejdwdTuraD3wYjuctAfYkGRF++B4A7CnLftOkvXtLKKrO9uSJI3AIIeJVgGfb2d7Lgf+W1X9ryQPAbcn2Qp8E3hfG383cAUwDXwP+ABAVR1N8jHgoTbuo1V1tE1/CLgVeB1wT7tIkkYkvRN4Tj6Tk5Pl/4Gsk8XE9rv4xg3vHncbOs0lebjz9YAf4zeQJUmGgSTJMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgXTcJrbfNe4WpKExDCRJhoEkyTCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kSCwiDJMuSfDnJn7X5tUkeTDKd5HNJzmj1M9v8dFs+0dnGta3+ZJLLOvWNrTadZPsQ758kaQALeWfwm8ATnfmPAzdW1c8CzwNbW30r8Hyr39jGkeQCYDPw88BG4I9bwCwDPglcDlwAvL+NlSSNyEBhkGQN8G7g020+wCXAHW3ITuDKNr2pzdOWX9rGbwJ2VdVLVfV1YBq4uF2mq+rpqvoBsKuNlSSNyKDvDP4I+D3gh23+DcALVfVymz8IrG7Tq4FnANryF9v4H9WPWWe2+qsk2ZZkKsnUkSNHBmxdkjSfecMgyT8CDlfVwyPoZ05VdXNVTVbV5MqVK8fdjiSdMpYPMObtwHuSXAG8FjgL+ARwdpLl7dX/GuBQG38IOB84mGQ58Hrg2536jO46s9UlSSMw7zuDqrq2qtZU1QS9D4Dvq6p/BtwPXNWGbQHubNO72zxt+X1VVa2+uZ1ttBZYB3wJeAhY185OOqPdxu6h3DtJ0kAGeWcwm98HdiX5A+DLwC2tfgvwmSTTwFF6T+5U1f4ktwOPAy8D11TVKwBJPgzsAZYBO6pq/wn0JUlaoAWFQVU9ADzQpp+mdybQsWO+D7x3lvWvB67vU78buHshvUiShsdvIEuSDANJkmEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIB2Xie13jbsFaagMA0mSYSBJMgwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhII+O3lrWUGQaSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCQxQBgkeW2SLyX5SpL9Sf5tq69N8mCS6SSfS3JGq5/Z5qfb8onOtq5t9SeTXNapb2y16STbF+F+SpLmMMg7g5eAS6rq7wEXAhuTrAc+DtxYVT8LPA9sbeO3As+3+o1tHEkuADYDPw9sBP44ybIky4BPApcDFwDvb2MlSSMybxhUz3fb7E+0SwGXAHe0+k7gyja9qc3Tll+aJK2+q6peqqqvA9PAxe0yXVVPV9UPgF1trCRpRAb6zKC9gn8EOAzsBZ4CXqiql9uQg8DqNr0aeAagLX8ReEO3fsw6s9UlSSMyUBhU1StVdSGwht4r+TcvZlOzSbItyVSSqSNHjoyjBUk6JS3obKKqegG4H/hF4Owky9uiNcChNn0IOB+gLX898O1u/Zh1Zqv3u/2bq2qyqiZXrly5kNYlSXMY5GyilUnObtOvA94FPEEvFK5qw7YAd7bp3W2etvy+qqpW39zONloLrAO+BDwErGtnJ51B70Pm3UO4b5KkAS2ffwjnATvbWT+vAW6vqj9L8jiwK8kfAF8GbmnjbwE+k2QaOErvyZ2q2p/kduBx4GXgmqp6BSDJh4E9wDJgR1XtH9o9lCTNa94wqKpHgbf0qT9N7/ODY+vfB947y7auB67vU78buHuAfiVJi8BvIEuSDANJkmEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAWbGL7XeNuQRo6w0CSZBhIkgwDSRKGgSQJw0CShGEgScIwkCRhGEgj5XcUtFQZBpIkw0CSZBhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJDBAGSc5Pcn+Sx5PsT/KbrX5Okr1JDrTrFa2eJDclmU7yaJKLOtva0sYfSLKlU39rksfaOjclyWLcWUlSf4O8M3gZ+J2qugBYD1yT5AJgO3BvVa0D7m3zAJcD69plG/Ap6IUHcB3wNuBi4LqZAGljPthZb+OJ3zVJ0qDmDYOqeraq/m+b/ivgCWA1sAnY2YbtBK5s05uA26pnH3B2kvOAy4C9VXW0qp4H9gIb27KzqmpfVRVwW2dbkqQRWNBnBkkmgLcADwKrqurZtuhbwKo2vRp4prPawVabq36wT73f7W9LMpVk6siRIwtpXZI0h4HDIMlPAX8K/FZVfae7rL2iryH39ipVdXNVTVbV5MqVKxf75iTptDFQGCT5CXpB8F+r6n+08nPtEA/t+nCrHwLO76y+ptXmqq/pU5ckjcggZxMFuAV4oqr+sLNoNzBzRtAW4M5O/ep2VtF64MV2OGkPsCHJivbB8QZgT1v2nSTr221d3dmWJGkElg8w5u3APwceS/JIq/1r4Abg9iRbgW8C72vL7gauAKaB7wEfAKiqo0k+BjzUxn20qo626Q8BtwKvA+5pF2nJ8f8w1qlq3jCoqv8DzHbe/6V9xhdwzSzb2gHs6FOfAn5hvl4kSYvDbyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQRs4fu9NSZBhIkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSANzJ+R0KnMMJAkGQaSJMNAkoRhII2Fnz9oqTEMJEmGgSTJMJAkYRhIkhggDJLsSHI4yVc7tXOS7E1yoF2vaPUkuSnJdJJHk1zUWWdLG38gyZZO/a1JHmvr3JQkw76TkqS5DfLO4FZg4zG17cC9VbUOuLfNA1wOrGuXbcCnoBcewHXA24CLgetmAqSN+WBnvWNvS5K0yOYNg6r6c+DoMeVNwM42vRO4slO/rXr2AWcnOQ+4DNhbVUer6nlgL7CxLTurqvZVVQG3dbYlSRqR4/3MYFVVPdumvwWsatOrgWc64w622lz1g33qfSXZlmQqydSRI0eOs3Vp4fxegE51J/wBcntFX0PoZZDburmqJqtqcuXKlaO4SUk6LRxvGDzXDvHQrg+3+iHg/M64Na02V31Nn7okaYSONwx2AzNnBG0B7uzUr25nFa0HXmyHk/YAG5KsaB8cbwD2tGXfSbK+nUV0dWdbkqQRGeTU0s8CXwR+LsnBJFuBG4B3JTkAvLPNA9wNPA1MA/8J+BBAVR0FPgY81C4fbTXamE+3dZ4C7hnOXZOWNj+H0FKyfL4BVfX+WRZd2mdsAdfMsp0dwI4+9SngF+brQ5K0ePwGsiTJMJAkGQaSJAwDaV5+0KvTgWEgSTIMpHHyXYeWCsNAkmQYSJIMA0kShoEkCcNAmpMf8Op0YRhIkgwDSZJhII2dh6K0FBgGkiTDQJJkGEiSMAykWY3yWL6fG2jcDANJkmEgSTIMpL7GcdjGQ0UaJ8NAkmQYSJIMA+lVxnm4xkNFGhfDQJJkGEhdS+GV+VLoQacfw0CSZBhIM5bSK/Kl1ItOD4aBtEQZCBql5eNuQBo3n3Ql3xnoNLfUg2Bi+11LvkedGgwDnbZOpifZk6lXnZyWzGGiJBuBTwDLgE9X1Q1jbkmnoJP5SXWm92/c8O4xd6JT0ZIIgyTLgE8C7wIOAg8l2V1Vj4+3M53sTuYn/9l075PBoGFZEmEAXAxMV9XTAEl2AZsAw0A/5lR8cj8Rc+0Pg0ILsVTCYDXwTGf+IPC2Ywcl2QZsa7PfTfLkcd7eucBfHue6i8m+FmbsfeXjfctj7wv69rYk+urDvhbmRPp642wLlkoYDKSqbgZuPtHtJJmqqskhtDRU9rUw9rUw9rUwp1tfS+VsokPA+Z35Na0mSRqBpRIGDwHrkqxNcgawGdg95p4k6bSxJA4TVdXLST4M7KF3aumOqtq/iDd5woeaFol9LYx9LYx9Lcxp1VeqajG2K0k6iSyVw0SSpDEyDCRJp24YJHlvkv1Jfphk1tOwkmxM8mSS6STbO/W1SR5s9c+1D7aH0dc5SfYmOdCuV/QZ844kj3Qu309yZVt2a5Kvd5ZdOKq+2rhXOre9u1Mf5/66MMkX29/70ST/tLNsqPtrtsdLZ/mZ7f5Pt/0x0Vl2bas/meSyE+njOPr67SSPt/1zb5I3dpb1/ZuOqK9fTXKkc/v/srNsS/u7H0iyZcR93djp6WtJXugsW5T9lWRHksNJvjrL8iS5qfX8aJKLOstOfF9V1Sl5Af4O8HPAA8DkLGOWAU8BbwLOAL4CXNCW3Q5sbtN/Avz6kPr698D2Nr0d+Pg8488BjgJ/o83fCly1CPtroL6A785SH9v+Av42sK5N/wzwLHD2sPfXXI+XzpgPAX/SpjcDn2vTF7TxZwJr23aWjbCvd3QeQ78+09dcf9MR9fWrwH/os+45wNPtekWbXjGqvo4Z/xv0TmpZ7P31D4CLgK/OsvwK4B4gwHrgwWHuq1P2nUFVPVFV831D+Uc/g1FVPwB2AZuSBLgEuKON2wlcOaTWNrXtDbrdq4B7qup7Q7r92Sy0rx8Z9/6qqq9V1YE2/RfAYWDlkG6/q+/jZY5+7wAubftnE7Crql6qqq8D0217I+mrqu7vPIb20fsuz2IbZH/N5jJgb1Udrarngb3AxjH19X7gs0O67VlV1Z/Te+E3m03AbdWzDzg7yXkMaV+dsmEwoH4/g7EaeAPwQlW9fEx9GFZV1bNt+lvAqnnGb+bVD8Tr29vEG5OcOeK+XptkKsm+mUNXLKH9leRieq/2nuqUh7W/Znu89B3T9seL9PbPIOsuZl9dW+m9wpzR7286yr7+Sfv73JFk5sunS2J/tcNpa4H7OuXF2l/zma3voeyrJfE9g+OV5AvAT/dZ9JGqunPU/cyYq6/uTFVVklnP7W2p/3fpff9ixrX0nhTPoHe+8e8DHx1hX2+sqkNJ3gTcl+Qxek94x23I++szwJaq+mErH/f+OhUl+RVgEvjlTvlVf9Oqeqr/FobufwKfraqXkvwreu+qLhnRbQ9iM3BHVb3SqY1zfy2akzoMquqdJ7iJ2X4G49v03oItb6/uFvTzGHP1leS5JOdV1bPtyevwHJt6H/D5qvrrzrZnXiW/lOQ/A787yr6q6lC7fjrJA8BbgD9lzPsryVnAXfReCOzrbPu491cfg/xsysyYg0mWA6+n93hazJ9cGWjbSd5JL2B/uapemqnP8jcdxpPbvH1V1bc7s5+m9xnRzLr/8Jh1HxhCTwP11bEZuKZbWMT9NZ/Z+h7KvjrdDxP1/RmM6n0qcz+94/UAW4BhvdPY3bY3yHZfdayyPSHOHKe/Euh75sFi9JVkxcxhliTnAm8HHh/3/mp/u8/TO556xzHLhrm/BvnZlG6/VwH3tf2zG9ic3tlGa4F1wJdOoJcF9ZXkLcB/BN5TVYc79b5/0xH2dV5n9j3AE216D7Ch9bcC2MCPv0Ne1L5ab2+m94HsFzu1xdxf89kNXN3OKloPvNhe7AxnXy3Gp+JL4QL8Y3rHzl4CngP2tPrPAHd3xl0BfI1esn+kU38TvX+s08B/B84cUl9vAO4FDgBfAM5p9Ul6/8PbzLgJeon/mmPWvw94jN6T2n8BfmpUfQG/1G77K+1661LYX8CvAH8NPNK5XLgY+6vf44XeYaf3tOnXtvs/3fbHmzrrfqSt9yRw+ZAf7/P19YX272Bm/+ye7286or7+HbC/3f79wJs76/6Lth+ngQ+Msq82/2+AG45Zb9H2F70Xfs+2x/JBep/t/Brwa2156P0nYE+1257srHvC+8qfo5AknfaHiSRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCQB/x8+68ScpRK0jQAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "_ = plt.hist(array[topk.indices], 1000, (-1, 1))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 171,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "11699132"
+ ]
+ },
+ "execution_count": 171,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(flat)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 172,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "3420.399391883936"
+ ]
+ },
+ "execution_count": 172,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sqrt(len(flat))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 180,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 181,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[2, 2, 191, 15313]"
+ ]
+ },
+ "execution_count": 181,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "prime_factors(len(flat))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 182,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "764"
+ ]
+ },
+ "execution_count": 182,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "2*2*191"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 183,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "square = flat.reshape(764, 15313)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 225,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(765, 15317)\n",
+ "(765, 15317)\n",
+ "torch.Size([764, 15313])\n",
+ "haar 46.458038330078125 125166.609375\n",
+ "(765, 15317)\n",
+ "(765, 15317)\n",
+ "torch.Size([764, 15313])\n",
+ "db1 46.458038330078125 125166.609375\n",
+ "(774, 15322)\n",
+ "(774, 15322)\n",
+ "torch.Size([764, 15313])\n",
+ "sym2 46.43218231201172 125249.15625\n",
+ "(781, 15331)\n",
+ "(781, 15331)\n",
+ "torch.Size([764, 15313])\n",
+ "coif1 46.54570388793945 125596.46875\n",
+ "(765, 15317)\n",
+ "(765, 15317)\n",
+ "torch.Size([764, 15313])\n",
+ "bior1.1 46.458038330078125 125166.609375\n",
+ "(765, 15317)\n",
+ "(765, 15317)\n",
+ "torch.Size([764, 15313])\n",
+ "rbio1.1 46.458038330078125 125166.609375\n",
+ "(1004, 15556)\n",
+ "(1004, 15556)\n",
+ "torch.Size([764, 15313])\n",
+ "dmey 52.37420654296875 139893.703125\n",
+ "(798, 15348)\n",
+ "(798, 15348)\n",
+ "torch.Size([764, 15313])\n",
+ "bior4.4 47.59955978393555 128068.578125\n",
+ "min: tensor(46.4322) sym2 2\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Base case on rectangular data, with level=4\n",
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "# wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " coeff = pywt.wavedecn(square.numpy(), wavelet, level = 4)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " shape = array.shape\n",
+ " print(shape)\n",
+ " array = torch.from_numpy(array).flatten()\n",
+ " topk = torch.topk(\n",
+ " array.abs(), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " \n",
+ " top10[topk.indices] = array[topk.indices]\n",
+ " top10 = top10.reshape(shape).numpy()\n",
+ " print(top10.shape)\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedecn\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverecn(og, wavelet = wavelet))\n",
+ " reverse_top10 = reverse_top10[0:, :-1]\n",
+ " print(reverse_top10.shape)\n",
+ " err = torch.norm(reverse_top10 - square, 2)\n",
+ " err1 = torch.norm(reverse_top10 - square, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 75,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# layerwise"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "stats = {k: l.shape for k,l in resw.items()}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'conv1.weight': torch.Size([64, 3, 7, 7]),\n",
+ " 'layer1.0.conv1.weight': torch.Size([64, 64, 3, 3]),\n",
+ " 'layer1.0.conv2.weight': torch.Size([64, 64, 3, 3]),\n",
+ " 'layer1.1.conv1.weight': torch.Size([64, 64, 3, 3]),\n",
+ " 'layer1.1.conv2.weight': torch.Size([64, 64, 3, 3]),\n",
+ " 'layer2.0.conv1.weight': torch.Size([128, 64, 3, 3]),\n",
+ " 'layer2.0.conv2.weight': torch.Size([128, 128, 3, 3]),\n",
+ " 'layer2.0.downsample.0.weight': torch.Size([128, 64, 1, 1]),\n",
+ " 'layer2.1.conv1.weight': torch.Size([128, 128, 3, 3]),\n",
+ " 'layer2.1.conv2.weight': torch.Size([128, 128, 3, 3]),\n",
+ " 'layer3.0.conv1.weight': torch.Size([256, 128, 3, 3]),\n",
+ " 'layer3.0.conv2.weight': torch.Size([256, 256, 3, 3]),\n",
+ " 'layer3.0.downsample.0.weight': torch.Size([256, 128, 1, 1]),\n",
+ " 'layer3.1.conv1.weight': torch.Size([256, 256, 3, 3]),\n",
+ " 'layer3.1.conv2.weight': torch.Size([256, 256, 3, 3]),\n",
+ " 'layer4.0.conv1.weight': torch.Size([512, 256, 3, 3]),\n",
+ " 'layer4.0.conv2.weight': torch.Size([512, 512, 3, 3]),\n",
+ " 'layer4.0.downsample.0.weight': torch.Size([512, 256, 1, 1]),\n",
+ " 'layer4.1.conv1.weight': torch.Size([512, 512, 3, 3]),\n",
+ " 'layer4.1.conv2.weight': torch.Size([512, 512, 3, 3]),\n",
+ " 'fc.weight': torch.Size([1000, 512])}"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "stats"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 78,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#r18.state_dict()['conv1.weight']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 79,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "torch.Size([64, 3, 7, 7])"
+ ]
+ },
+ "execution_count": 79,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "r18.state_dict()['conv1.weight'].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 596,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "res = pywt.wavedec(r18.state_dict()['conv1.weight'].numpy(), \"sym2\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 597,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(64, 3, 7, 5)"
+ ]
+ },
+ "execution_count": 597,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "res[1].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 598,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(64, 3, 7, 5)"
+ ]
+ },
+ "execution_count": 598,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "res[0].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 600,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_weight = r18.state_dict()['conv1.weight'].numpy()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 602,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor([-0.0104, -0.0061, -0.0018, ..., -0.0244, -0.0712, -0.0668])"
+ ]
+ },
+ "execution_count": 602,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "flatten_model({\"f1\": r18.state_dict()['conv1.weight']})"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 605,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat_v1w = flatten_model({\"f1\": r18.state_dict()['conv1.weight']}).numpy()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 618,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 630,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "torch.Size([])"
+ ]
+ },
+ "execution_count": 630,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# resw[\"bn1.num_batches_tracked\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 47.194419860839844 124593.125\n",
+ "db1 47.194419860839844 124593.125\n",
+ "sym2 47.194419860839844 124593.125\n",
+ "coif1 47.194419860839844 124593.125\n",
+ "bior1.1 47.194419860839844 124593.125\n",
+ "rbio1.1 47.194419860839844 124593.125\n",
+ "dmey 47.194419860839844 124593.125\n",
+ "bior4.4 47.194419860839844 124593.125\n",
+ "min: tensor(47.1944) haar 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavedecn, with level = 0\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "#wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " shape = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " coeffs = []\n",
+ " to_del = []\n",
+ " for key, v in resw.items():\n",
+ " #print(key, v.shape)\n",
+ " if v.shape == torch.Size([]):\n",
+ " print(key)\n",
+ " to_del.append(key)\n",
+ " continue\n",
+ " \n",
+ " coeff = pywt.wavedecn(torch.squeeze(v).numpy(), wavelet, level = 0)\n",
+ " #print(coeff[0].shape)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " shape.append(array.shape)\n",
+ " coeffs.append(coeff_slices)\n",
+ " #print(array.shape)\n",
+ " flat_array = torch.from_numpy(array).flatten()\n",
+ " lens.append(len(flat_array))\n",
+ " to_cat.append(flat_array)\n",
+ " for k in to_del:\n",
+ " del resw[k]\n",
+ " flat_wv = torch.cat(to_cat)\n",
+ " topk = torch.topk(\n",
+ " flat_wv.abs(), round(0.1*len(flat_wv)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_wv))\n",
+ " top10[topk.indices] = flat_wv[topk.indices]\n",
+ " \n",
+ " start_index = 0\n",
+ " state_dict = {}\n",
+ " for i, key in enumerate(resw):\n",
+ " end_index = start_index + lens[i]\n",
+ " #print(start_index, end_index, top10.shape)\n",
+ " #print(top10)\n",
+ " crr = top10[start_index:end_index]\n",
+ " #print(crr.shape)\n",
+ " og = pywt.array_to_coeffs(crr.reshape(shape[i]), coeffs[i], output_format=\"wavedecn\")\n",
+ " state_dict[key] = torch.from_numpy(pywt.waverecn(og, wavelet = wavelet))\n",
+ " start_index = end_index\n",
+ "\n",
+ " reverse_top10 = flatten_model(state_dict)\n",
+ " flat = flatten_model(resw)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 110,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[(64, 3, 7, 7),\n",
+ " (64, 64, 3, 3),\n",
+ " (64, 64, 3, 3),\n",
+ " (64, 64, 3, 3),\n",
+ " (64, 64, 3, 3),\n",
+ " (128, 64, 3, 3),\n",
+ " (128, 128, 3, 3),\n",
+ " (128, 64),\n",
+ " (128, 128, 3, 3),\n",
+ " (128, 128, 3, 3),\n",
+ " (256, 128, 3, 3),\n",
+ " (256, 256, 3, 3),\n",
+ " (256, 128),\n",
+ " (256, 256, 3, 3),\n",
+ " (256, 256, 3, 3),\n",
+ " (512, 256, 3, 3),\n",
+ " (512, 512, 3, 3),\n",
+ " (512, 256),\n",
+ " (512, 512, 3, 3),\n",
+ " (512, 512, 3, 3),\n",
+ " (1000, 512)]"
+ ]
+ },
+ "execution_count": 110,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## wavedecn does not work corretly for dim > 2, so we need to reshape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 152,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "resw_oe = resw.copy()\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 153,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "del resw_oe[\"conv1.weight\"] # for this one the size is incorrect during reconstruction"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 227,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "11669504 11669504\n",
+ "haar 45.01624298095703 121187.4765625\n",
+ "11669504 11669504\n",
+ "db1 45.01624298095703 121187.4765625\n",
+ "11669504 11669504\n",
+ "sym2 44.518096923828125 120046.953125\n",
+ "11669504 11669504\n",
+ "coif1 44.400272369384766 119750.8671875\n",
+ "11669504 11669504\n",
+ "bior1.1 45.01624298095703 121187.4765625\n",
+ "11669504 11669504\n",
+ "rbio1.1 45.01624298095703 121187.4765625\n",
+ "11669504 11669504\n",
+ "dmey 44.74907302856445 120040.7890625\n",
+ "11669504 11669504\n",
+ "bior4.4 44.85655975341797 120709.2578125\n",
+ "min: tensor(44.4003) coif1 3\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavedecn with axes specified, on dimensions reduced to 2x2\n",
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "#wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " shape = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " coeffs = []\n",
+ " to_del = []\n",
+ " for key, v in resw_oe.items():\n",
+ " #print(key, v.shape)\n",
+ " if v.shape == torch.Size([]):\n",
+ " print(key)\n",
+ " to_del.append(key)\n",
+ " continue\n",
+ " v = torch.squeeze(v).numpy()\n",
+ " if len(v.shape) > 2:\n",
+ " #print(v.shape)\n",
+ " v = v.reshape((v.shape[0], np.prod(v.shape[1:])))\n",
+ " #print(v.shape)\n",
+ " coeff = pywt.wavedecn(v, wavelet, level = None, axes = (0,1))\n",
+ " #print(coeff[0].shape)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff, axes = (0,1))\n",
+ " shape.append(array.shape)\n",
+ " coeffs.append(coeff_slices)\n",
+ " #print(array.shape)\n",
+ " flat_array = torch.from_numpy(array).flatten()\n",
+ " lens.append(len(flat_array))\n",
+ " to_cat.append(flat_array)\n",
+ " for k in to_del:\n",
+ " del resw_oe[k]\n",
+ " flat_wv = torch.cat(to_cat)\n",
+ " topk = torch.topk(\n",
+ " flat_wv.abs(), round(0.1*len(flat_wv)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_wv))\n",
+ " top10[topk.indices] = flat_wv[topk.indices]\n",
+ " \n",
+ " start_index = 0\n",
+ " state_dict = {}\n",
+ " for i, key in enumerate(resw_oe):\n",
+ " end_index = start_index + lens[i]\n",
+ " #print(start_index, end_index, top10.shape)\n",
+ " #print(top10)\n",
+ " crr = top10[start_index:end_index]\n",
+ " #print(crr.shape)\n",
+ " og = pywt.array_to_coeffs(crr.reshape(shape[i]), coeffs[i], output_format=\"wavedecn\")\n",
+ " state_dict[key] = torch.from_numpy(pywt.waverecn(og, wavelet = wavelet, axes = (0,1)))\n",
+ " # print(state_dict[key].shape, shape[i])\n",
+ " #assert np.prod(state_dict[key].shape) == np.prod(shape[i])\n",
+ " start_index = end_index\n",
+ "\n",
+ " reverse_top10 = flatten_model(state_dict)\n",
+ " \n",
+ " flat = flatten_model(resw_oe)\n",
+ " for k, v in resw_oe.items():\n",
+ " # print(k, v.shape, state_dict[k].shape)\n",
+ " assert np.prod(torch.squeeze(v).shape) == np.prod(state_dict[k].shape)\n",
+ " print(len(reverse_top10), len(flat))\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 235,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "11669504 11669504\n",
+ "haar 47.135135650634766 124408.8359375\n",
+ "11669504 11669504\n",
+ "db1 47.135135650634766 124408.8359375\n",
+ "11669504 11669504\n",
+ "sym2 47.135135650634766 124408.8359375\n",
+ "11669504 11669504\n",
+ "coif1 47.135135650634766 124408.8359375\n",
+ "11669504 11669504\n",
+ "bior1.1 47.135135650634766 124408.8359375\n",
+ "11669504 11669504\n",
+ "rbio1.1 47.135135650634766 124408.8359375\n",
+ "11669504 11669504\n",
+ "dmey 47.135135650634766 124408.8359375\n",
+ "11669504 11669504\n",
+ "bior4.4 47.135135650634766 124408.8359375\n",
+ "min: tensor(47.1351) haar 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavedecn with axes specified, on dimensions reduced to 2x2, level = 0\n",
+ "\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "#wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " shape = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " coeffs = []\n",
+ " to_del = []\n",
+ " for key, v in resw_oe.items():\n",
+ " #print(key, v.shape)\n",
+ " if v.shape == torch.Size([]):\n",
+ " print(key)\n",
+ " to_del.append(key)\n",
+ " continue\n",
+ " v = torch.squeeze(v).numpy()\n",
+ " if len(v.shape) > 2:\n",
+ " #print(v.shape)\n",
+ " v = v.reshape((v.shape[0], np.prod(v.shape[1:])))\n",
+ " #print(v.shape)\n",
+ " coeff = pywt.wavedecn(v, wavelet, level = 0, axes = (0,1))\n",
+ " #print(coeff[0].shape)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff, axes = (0,1))\n",
+ " shape.append(array.shape)\n",
+ " coeffs.append(coeff_slices)\n",
+ " #print(array.shape)\n",
+ " flat_array = torch.from_numpy(array).flatten()\n",
+ " lens.append(len(flat_array))\n",
+ " to_cat.append(flat_array)\n",
+ " for k in to_del:\n",
+ " del resw_oe[k]\n",
+ " flat_wv = torch.cat(to_cat)\n",
+ " topk = torch.topk(\n",
+ " flat_wv.abs(), round(0.1*len(flat_wv)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_wv))\n",
+ " top10[topk.indices] = flat_wv[topk.indices]\n",
+ " \n",
+ " start_index = 0\n",
+ " state_dict = {}\n",
+ " for i, key in enumerate(resw_oe):\n",
+ " end_index = start_index + lens[i]\n",
+ " #print(start_index, end_index, top10.shape)\n",
+ " #print(top10)\n",
+ " crr = top10[start_index:end_index]\n",
+ " #print(crr.shape)\n",
+ " og = pywt.array_to_coeffs(crr.reshape(shape[i]), coeffs[i], output_format=\"wavedecn\")\n",
+ " state_dict[key] = torch.from_numpy(pywt.waverecn(og, wavelet = wavelet, axes = (0,1)))\n",
+ " # print(state_dict[key].shape, shape[i])\n",
+ " #assert np.prod(state_dict[key].shape) == np.prod(shape[i])\n",
+ " start_index = end_index\n",
+ "\n",
+ " reverse_top10 = flatten_model(state_dict)\n",
+ " \n",
+ " flat = flatten_model(resw_oe)\n",
+ " for k, v in resw_oe.items():\n",
+ " # print(k, v.shape, state_dict[k].shape)\n",
+ " assert np.prod(torch.squeeze(v).shape) == np.prod(state_dict[k].shape)\n",
+ " print(len(reverse_top10), len(flat))\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 228,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 47.194419860839844 124593.125\n",
+ "db1 47.194419860839844 124593.125\n",
+ "sym2 47.194419860839844 124593.125\n",
+ "coif1 47.194419860839844 124593.125\n",
+ "bior1.1 47.194419860839844 124593.125\n",
+ "rbio1.1 47.194419860839844 124593.125\n",
+ "dmey 47.194419860839844 124593.125\n",
+ "bior4.4 47.194419860839844 124593.125\n",
+ "min: tensor(47.1944) haar 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavedecn implementation with axes\n",
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "#wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " shape = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " coeffs = []\n",
+ " to_del = []\n",
+ " for key, v in resw.items():\n",
+ " #print(key, v.shape)\n",
+ " if v.shape == torch.Size([]):\n",
+ " print(key)\n",
+ " to_del.append(key)\n",
+ " continue\n",
+ " \n",
+ " coeff = pywt.wavedecn(torch.squeeze(v).numpy(), wavelet, level = 0, axes = (0,1))\n",
+ " #print(coeff[0].shape)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff, axes = (0,1))\n",
+ " shape.append(array.shape)\n",
+ " coeffs.append(coeff_slices)\n",
+ " #print(array.shape)\n",
+ " flat_array = torch.from_numpy(array).flatten()\n",
+ " lens.append(len(flat_array))\n",
+ " to_cat.append(flat_array)\n",
+ " for k in to_del:\n",
+ " del resw[k]\n",
+ " flat_wv = torch.cat(to_cat)\n",
+ " topk = torch.topk(\n",
+ " flat_wv.abs(), round(0.1*len(flat_wv)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_wv))\n",
+ " top10[topk.indices] = flat_wv[topk.indices]\n",
+ " \n",
+ " start_index = 0\n",
+ " state_dict = {}\n",
+ " for i, key in enumerate(resw):\n",
+ " end_index = start_index + lens[i]\n",
+ " #print(start_index, end_index, top10.shape)\n",
+ " #print(top10)\n",
+ " crr = top10[start_index:end_index]\n",
+ " #print(crr.shape)\n",
+ " og = pywt.array_to_coeffs(crr.reshape(shape[i]), coeffs[i], output_format=\"wavedec\")\n",
+ " state_dict[key] = torch.from_numpy(pywt.waverecn(og, wavelet = wavelet, axes = (0,1)))\n",
+ " start_index = end_index\n",
+ "\n",
+ " reverse_top10 = flatten_model(state_dict)\n",
+ " flat = flatten_model(resw)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 229,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 47.194419860839844 124593.125\n",
+ "db1 47.194419860839844 124593.125\n",
+ "sym2 47.194419860839844 124593.125\n",
+ "coif1 47.194419860839844 124593.125\n",
+ "bior1.1 47.194419860839844 124593.125\n",
+ "rbio1.1 47.194419860839844 124593.125\n",
+ "dmey 47.194419860839844 124593.125\n",
+ "bior4.4 47.194419860839844 124593.125\n",
+ "min: tensor(47.1944) haar 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavedec on flat\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "#wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " shape = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " coeffs = []\n",
+ " to_del = []\n",
+ " for key, v in resw.items():\n",
+ " #print(key, v.shape)\n",
+ " if v.shape == torch.Size([]):\n",
+ " print(key)\n",
+ " to_del.append(key)\n",
+ " continue\n",
+ " \n",
+ " coeff = pywt.wavedec(torch.squeeze(v.flatten()).numpy(), wavelet, level = 0)\n",
+ " #print(coeff[0].shape)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " shape.append(array.shape)\n",
+ " coeffs.append(coeff_slices)\n",
+ " #print(array.shape)\n",
+ " flat_array = torch.from_numpy(array).flatten()\n",
+ " lens.append(len(flat_array))\n",
+ " to_cat.append(flat_array)\n",
+ " for k in to_del:\n",
+ " del resw[k]\n",
+ " flat_wv = torch.cat(to_cat)\n",
+ " topk = torch.topk(\n",
+ " flat_wv.abs(), round(0.1*len(flat_wv)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_wv))\n",
+ " top10[topk.indices] = flat_wv[topk.indices]\n",
+ " \n",
+ " start_index = 0\n",
+ " state_dict = {}\n",
+ " for i, key in enumerate(resw):\n",
+ " end_index = start_index + lens[i]\n",
+ " #print(start_index, end_index, top10.shape)\n",
+ " #print(top10)\n",
+ " crr = top10[start_index:end_index]\n",
+ " #print(crr.shape)\n",
+ " og = pywt.array_to_coeffs(crr.reshape(shape[i]), coeffs[i], output_format=\"wavedec\")\n",
+ " state_dict[key] = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " start_index = end_index\n",
+ "\n",
+ " reverse_top10 = flatten_model(state_dict)\n",
+ " flat = flatten_model(resw)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 230,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 42.886417388916016 114614.34375\n",
+ "db1 42.886417388916016 114614.34375\n",
+ "sym2 42.55022430419922 113962.6484375\n",
+ "coif1 42.59132385253906 114232.9609375\n",
+ "bior1.1 42.886417388916016 114614.34375\n",
+ "rbio1.1 42.886417388916016 114614.34375\n",
+ "dmey 42.56193542480469 113790.9296875\n",
+ "bior4.4 42.609718322753906 114087.71875\n",
+ "min: tensor(42.5502) sym2 2\n"
+ ]
+ }
+ ],
+ "source": [
+ "# manual layerwise wavelet with level = None\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "#wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " shape = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " coeffs = []\n",
+ " to_del = []\n",
+ " for key, v in resw.items():\n",
+ " #print(key, v.shape)\n",
+ " if v.shape == torch.Size([]):\n",
+ " print(key)\n",
+ " to_del.append(key)\n",
+ " continue\n",
+ " \n",
+ " coeff = pywt.wavedec(torch.squeeze(v.flatten()).numpy(), wavelet, level = None)\n",
+ " #print(coeff[0].shape)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " shape.append(array.shape)\n",
+ " coeffs.append(coeff_slices)\n",
+ " #print(array.shape)\n",
+ " flat_array = torch.from_numpy(array).flatten()\n",
+ " lens.append(len(flat_array))\n",
+ " to_cat.append(flat_array)\n",
+ " for k in to_del:\n",
+ " del resw[k]\n",
+ " flat_wv = torch.cat(to_cat)\n",
+ " topk = torch.topk(\n",
+ " flat_wv.abs(), round(0.1*len(flat_wv)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_wv))\n",
+ " top10[topk.indices] = flat_wv[topk.indices]\n",
+ " \n",
+ " start_index = 0\n",
+ " state_dict = {}\n",
+ " for i, key in enumerate(resw):\n",
+ " end_index = start_index + lens[i]\n",
+ " #print(start_index, end_index, top10.shape)\n",
+ " #print(top10)\n",
+ " crr = top10[start_index:end_index]\n",
+ " #print(crr.shape)\n",
+ " og = pywt.array_to_coeffs(crr.reshape(shape[i]), coeffs[i], output_format=\"wavedec\")\n",
+ " state_dict[key] = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " start_index = end_index\n",
+ "\n",
+ " reverse_top10 = flatten_model(state_dict)\n",
+ " flat = flatten_model(resw)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 62,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "resw2 = {}\n",
+ "resw2[\"fc.weight\"] = resw[\"fc.weight\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 63,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "resw = resw2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 64,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "torch.Size([1000, 512])"
+ ]
+ },
+ "execution_count": 64,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "resw2[\"fc.weight\"].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 65,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 31.85331916809082 18000.552734375\n",
+ "db1 31.85331916809082 18000.552734375\n",
+ "sym2 31.85331916809082 18000.552734375\n",
+ "coif1 31.85331916809082 18000.552734375\n",
+ "bior1.1 31.85331916809082 18000.552734375\n",
+ "rbio1.1 31.85331916809082 18000.552734375\n",
+ "dmey 31.85331916809082 18000.552734375\n",
+ "bior4.4 31.85331916809082 18000.552734375\n",
+ "min: tensor(31.8533) haar 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavedecn implementation with axes\n",
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "#wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " lens = []\n",
+ " shape = []\n",
+ " fft_layers = []\n",
+ " to_cat = []\n",
+ " coeffs = []\n",
+ " to_del = []\n",
+ " for key, v in resw.items():\n",
+ " #print(key, v.shape)\n",
+ " if v.shape == torch.Size([]):\n",
+ " print(key)\n",
+ " to_del.append(key)\n",
+ " continue\n",
+ " \n",
+ " coeff = pywt.wavedecn(torch.squeeze(v).numpy(), wavelet, level = 0, axes = (0,1))\n",
+ " #print(coeff[0].shape)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff, axes = (0,1))\n",
+ " shape.append(array.shape)\n",
+ " coeffs.append(coeff_slices)\n",
+ " #print(array.shape)\n",
+ " flat_array = torch.from_numpy(array).flatten()\n",
+ " lens.append(len(flat_array))\n",
+ " to_cat.append(flat_array)\n",
+ " for k in to_del:\n",
+ " del resw[k]\n",
+ " flat_wv = torch.cat(to_cat)\n",
+ " topk = torch.topk(\n",
+ " flat_wv.abs(), round(0.1*len(flat_wv)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(flat_wv))\n",
+ " top10[topk.indices] = flat_wv[topk.indices]\n",
+ " \n",
+ " start_index = 0\n",
+ " state_dict = {}\n",
+ " for i, key in enumerate(resw):\n",
+ " end_index = start_index + lens[i]\n",
+ " #print(start_index, end_index, top10.shape)\n",
+ " #print(top10)\n",
+ " crr = top10[start_index:end_index]\n",
+ " #print(crr.shape)\n",
+ " og = pywt.array_to_coeffs(crr.reshape(shape[i]), coeffs[i], output_format=\"wavedec\")\n",
+ " state_dict[key] = torch.from_numpy(pywt.waverecn(og, wavelet = wavelet, axes = (0,1)))\n",
+ " start_index = end_index\n",
+ "\n",
+ " reverse_top10 = flatten_model(state_dict)\n",
+ " flat = flatten_model(resw)\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 68,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat = flatten_model(resw)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 69,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ "top10_og = torch.zeros(len(flat))\n",
+ "top10_og[topk_og.indices] = flat[topk_og.indices]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 70,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(31.8533)"
+ ]
+ },
+ "execution_count": 70,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(top10_og - flat, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 71,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 34.37525939941406 19823.703125\n",
+ "db1 34.37525939941406 19823.703125\n",
+ "sym2 34.24169921875 19703.4453125\n",
+ "coif1 34.238433837890625 19669.91796875\n",
+ "bior1.1 34.37525939941406 19823.703125\n",
+ "rbio1.1 34.37525939941406 19823.703125\n",
+ "dmey 35.10240936279297 20091.73828125\n",
+ "bior4.4 34.86445999145508 19995.5078125\n",
+ "min: tensor(34.2384) coif1 3\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavelet on flattened\n",
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "# wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = 4)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 74,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "not_flat = list(resw.values())[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 87,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "torch.Size([1000, 512])"
+ ]
+ },
+ "execution_count": 87,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "not_flat.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 233,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "haar 31.85331916809082 18000.552734375\n",
+ "db1 31.85331916809082 18000.552734375\n",
+ "sym2 31.85331916809082 18000.552734375\n",
+ "coif1 31.85331916809082 18000.552734375\n",
+ "bior1.1 31.85331916809082 18000.552734375\n",
+ "rbio1.1 31.85331916809082 18000.552734375\n",
+ "dmey 31.85331916809082 18000.552734375\n",
+ "bior4.4 31.85331916809082 18000.552734375\n",
+ "min: tensor(31.8533) haar 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavdecn on not flat, with level None\n",
+ "# working on the random initialization\n",
+ "wavelets = ['haar', 'db1', 'sym2', 'coif1', 'bior1.1', 'rbio1.1', 'dmey', 'bior4.4'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "# wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " coeff = pywt.wavedecn(not_flat.numpy(), wavelet, level = 0)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " shape = array.shape\n",
+ " array = array.flatten()\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " top10 = top10.reshape(shape)\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedecn\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverecn(og, wavelet = wavelet))\n",
+ " err = torch.norm(reverse_top10 - not_flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - not_flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "Wavelet create an array of the entire thing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Combination of the best"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "resw2 = {}\n",
+ "resw2[\"fc.weight\"] = resw[\"fc.weight\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "del resw[\"fc.weight\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat = flatten_model(resw)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "sym2 38.345680236816406 100879.03125\n",
+ "min: tensor(38.3457) sym2 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Base case on flattened data, with level=4\n",
+ "# working on the random initialization\n",
+ "wavelets = ['sym2'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "# wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " coeff = pywt.wavedec(flat.numpy(), wavelet, level = 4)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " #print(coeff_slices) # should be static so we do not need to send them\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedec\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverec(og, wavelet = wavelet))\n",
+ " err = torch.norm(reverse_top10 - flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "r10_1 = reverse_top10"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(43.5638)"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " flat.abs(), round(0.1*len(flat)), dim=0, sorted=False\n",
+ " )\n",
+ "top10_og1 = torch.zeros(len(flat))\n",
+ "top10_og1[topk_og.indices] = flat[topk_og.indices]\n",
+ "torch.norm(top10_og1 - flat, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "not_flat = list(resw2.values())[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "sym2 31.85331916809082 18000.552734375\n",
+ "min: tensor(31.8533) sym2 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# wavdecn on not flat, with level None\n",
+ "# working on the random initialization\n",
+ "wavelets = ['sym2'] # 'gaus1' not supported, 'mexh','morl', 'cgau1', 'shan', 'fbsp', 'cmor'\n",
+ "# wavelets = pywt.wavelist(kind='discrete', )\n",
+ "errs = []\n",
+ "names = []\n",
+ "for wavelet in wavelets:\n",
+ " coeff = pywt.wavedecn(not_flat.numpy(), wavelet, level = 0)\n",
+ " array, coeff_slices = pywt.coeffs_to_array(coeff)\n",
+ " shape = array.shape\n",
+ " array = array.flatten()\n",
+ " topk = torch.topk(\n",
+ " torch.from_numpy(np.absolute(array)), round(0.1*len(array)), dim=0, sorted=False\n",
+ " )\n",
+ " top10 = torch.zeros(len(array))\n",
+ " top10[topk.indices] = torch.from_numpy(array[topk.indices])\n",
+ " top10 = top10.reshape(shape)\n",
+ " og = pywt.array_to_coeffs(top10, coeff_slices, output_format=\"wavedecn\")\n",
+ " reverse_top10 = torch.from_numpy(pywt.waverecn(og, wavelet = wavelet))\n",
+ " err = torch.norm(reverse_top10 - not_flat, 2)\n",
+ " err1 = torch.norm(reverse_top10 - not_flat, 1)\n",
+ " errs.append(err)\n",
+ " names.append(wavelet)\n",
+ " print(wavelet, err.item(), err1.item())\n",
+ "ind = np.argmin(errs)\n",
+ "print(\"min: \", errs[ind], names[ind], ind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "r10 = torch.cat([r10_1, reverse_top10.flatten()])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(50.8493)"
+ ]
+ },
+ "execution_count": 33,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "topk_og = torch.topk(\n",
+ " not_flat.flatten().abs(), round(0.1*len(not_flat.flatten())), dim=0, sorted=False\n",
+ " )\n",
+ "top10_og2 = torch.zeros(len(not_flat.flatten()))\n",
+ "top10_og2[topk_og.indices] = flat[topk_og.indices]\n",
+ "torch.norm(top10_og2 - not_flat.flatten(), 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "r10_og = torch.cat([top10_og1, top10_og2])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "models = torchvision.models.resnet18(True).state_dict()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "models = {k:v for k,v in models.items() if \"bn1.\" not in k}\n",
+ "models = {k:v for k,v in models.items() if \"bn2.\" not in k}\n",
+ "models = {k:v for k,v in models.items() if len(v.shape) >1}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 46,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flat= flatten_model(models)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 47,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(49.8481)"
+ ]
+ },
+ "execution_count": 47,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(r10 - flat, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(66.9528)"
+ ]
+ },
+ "execution_count": 48,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.norm(r10_og - flat, 2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "interpreter": {
+ "hash": "996934296aa9d79be6c3d800a38d8fdb7dfa8fe7bb07df178f1397cde2cb8742"
+ },
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/random files/plotting_from_csv.py b/random files/plotting_from_csv.py
new file mode 100644
index 0000000000000000000000000000000000000000..07f89a0e26b9881c1f3293be4fb88d806d35eab0
--- /dev/null
+++ b/random files/plotting_from_csv.py
@@ -0,0 +1,173 @@
+import distutils
+import json
+import os
+import sys
+
+import numpy as np
+import pandas as pd
+from matplotlib import pyplot as plt
+
+
+def plot(x_axis, means, stdevs, pos, nb_plots, title, label, loc, xlabel):
+ cmap = plt.get_cmap("gist_rainbow")
+ plt.title(title)
+ plt.xlabel(xlabel)
+ y_axis = list(means)
+ err = list(stdevs)
+ print("label:", label)
+ print("color: ", cmap(1 / nb_plots * pos))
+ plt.errorbar(
+ list(x_axis), y_axis, yerr=err, label=label, color=cmap(1 / nb_plots * pos)
+ )
+ plt.legend(loc=loc)
+
+
+def plot_results(path, epochs, global_epochs=True):
+ print(path, epochs, global_epochs, type(global_epochs))
+ global_epochs = bool(distutils.util.strtobool(global_epochs))
+ epochs = int(epochs)
+ folders = os.listdir(path)
+ folders.sort()
+ print("Reading folders from: ", path)
+ print("Folders: ", folders)
+ bytes_means, bytes_stdevs = {}, {}
+ meta_means, meta_stdevs = {}, {}
+ data_means, data_stdevs = {}, {}
+
+ files = os.listdir(path)
+ files = [f for f in files if f.endswith(".csv")]
+ train_loss = sorted([f for f in files if f.startswith("train_loss")])
+ test_acc = sorted([f for f in files if f.startswith("test_acc")])
+ test_loss = sorted([f for f in files if f.startswith("test_loss")])
+ max_losses = []
+ for i, f in enumerate(train_loss):
+ filepath = os.path.join(path, f)
+ with open(filepath, "r") as inf:
+ results_csv = pd.read_csv(inf)
+ # Plot Training loss
+ plt.figure(1)
+ if global_epochs:
+ means = results_csv["mean"].to_numpy()
+ stdevs = results_csv["std"].to_numpy()
+ means = means[:epochs]
+ stdevs = stdevs[:epochs]
+ x_axis = list(np.arange(0, len(means), 1))
+ x_label = "global epochs"
+ else:
+ results_cr = results_csv[results_csv.rounds <= epochs]
+ means = results_cr["mean"].to_numpy()
+ stdevs = results_cr["std"].to_numpy()
+ x_axis = results_cr["rounds"].to_numpy()
+ x_label = "communication rounds"
+ max_losses.append(np.max(means))
+
+ plot(
+ x_axis,
+ means,
+ stdevs,
+ i,
+ len(train_loss),
+ "Training Loss",
+ f[len("train_loss") + 1 : -len(":2022-03-24T17:54.csv")],
+ "upper right",
+ x_label,
+ )
+
+ max_tlosses = []
+ for i, f in enumerate(test_loss):
+ filepath = os.path.join(path, f)
+ with open(filepath, "r") as inf:
+ results_csv = pd.read_csv(inf)
+ if global_epochs:
+ means = results_csv["mean"].to_numpy()
+ stdevs = results_csv["std"].to_numpy()
+ means = means[:epochs]
+ stdevs = stdevs[:epochs]
+ x_axis = list(np.arange(0, len(means), 1))
+ x_label = "global epochs"
+ else:
+ results_cr = results_csv[results_csv.rounds <= epochs]
+ means = results_cr["mean"].to_numpy()
+ stdevs = results_cr["std"].to_numpy()
+ x_axis = results_cr["rounds"].to_numpy()
+ x_label = "communication rounds"
+ print("x axis:", x_axis)
+ max_tlosses.append(np.max(means))
+ # Plot Testing loss
+ plt.figure(2)
+ plot(
+ x_axis,
+ means,
+ stdevs,
+ i,
+ len(test_loss),
+ "Testing Loss",
+ f[len("test_loss") + 1 : -len(":2022-03-24T17:54.csv")],
+ "upper right",
+ x_label,
+ )
+
+ max_taccs = []
+ for i, f in enumerate(test_acc):
+ filepath = os.path.join(path, f)
+ with open(filepath, "r") as inf:
+ results_csv = pd.read_csv(inf)
+ if global_epochs:
+ means = results_csv["mean"].to_numpy()
+ stdevs = results_csv["std"].to_numpy()
+ means = means[:epochs]
+ stdevs = stdevs[:epochs]
+ x_axis = list(np.arange(0, len(means), 1))
+ x_label = "global epochs"
+ else:
+ results_cr = results_csv[results_csv.rounds <= epochs]
+ means = results_cr["mean"].to_numpy()
+ stdevs = results_cr["std"].to_numpy()
+ x_axis = results_cr["rounds"].to_numpy()
+ x_label = "communication rounds"
+ max_taccs.append(np.max(means))
+ # Plot Testing Accuracy
+ plt.figure(3)
+ plot(
+ x_axis,
+ means,
+ stdevs,
+ i,
+ len(test_acc),
+ "Testing Accuracy",
+ f[len("test_acc") + 1 : -len(":2022-03-24T17:54.csv")],
+ "lower right",
+ x_label,
+ )
+
+ names_loss = [
+ f[len("train_loss") + 1 : -len(":2022-03-24T17:54.csv")] for f in train_loss
+ ]
+ names_acc = [
+ f[len("test_acc") + 1 : -len(":2022-03-24T17:54.csv")] for f in test_acc
+ ]
+ print(names_loss)
+ print(names_acc)
+ pf = pd.DataFrame(
+ {
+ "test_accuracy": max_taccs,
+ "test_losses": max_tlosses,
+ "train_losses": max_losses,
+ },
+ names_loss,
+ )
+ pf = pf.sort_values(["test_accuracy"], 0, ascending=False)
+ pf.to_csv(os.path.join(path, "best_results.csv"))
+
+ plt.figure(1)
+ plt.savefig(os.path.join(path, "ge_train_loss.png"), dpi=300)
+ plt.figure(2)
+ plt.savefig(os.path.join(path, "ge_test_loss.png"), dpi=300)
+ plt.figure(3)
+ plt.savefig(os.path.join(path, "ge_test_acc.png"), dpi=300)
+
+
+if __name__ == "__main__":
+ assert len(sys.argv) == 4
+ print(sys.argv[1], sys.argv[2], sys.argv[3])
+ plot_results(sys.argv[1], sys.argv[2], sys.argv[3])
diff --git a/random files/reddit.ipynb b/random files/reddit.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..b3496c608e081082273e686f34f8521e6eef7c8d
--- /dev/null
+++ b/random files/reddit.ipynb
@@ -0,0 +1,960 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "ZMZYcW3itMzT",
+ "outputId": "f2970f7e-cf26-4a67-e8d3-29bcd1a11775"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "2VftlLfttdT8",
+ "outputId": "48b47fdc-853b-4711-ae95-8c0e64510615"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "ft7BMl1LyWP6"
+ },
+ "outputs": [],
+ "source": [
+ "from torch import nn\n",
+ "import torch\n",
+ "import os\n",
+ "import json\n",
+ "import pickle\n",
+ "import numpy as np\n",
+ "import pywt\n",
+ "import collections\n",
+ "from decentralizepy.datasets.Partitioner import DataPartitioner\n",
+ "from collections import defaultdict\n",
+ "train_dir = \"/home/jeffrey/Downloads/reddit/per_user_data/train\"\n",
+ "test_dir=\"/home/jeffrey/Downloads/reddit/new_small_data/test\"\n",
+ "my_train_data = {\"x\": [], \"y\": []}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def test_reddit(tmpdir):\n",
+ " mapping = Linear(6, 16)\n",
+ " reddit = Reddit(0,0,mapping,\n",
+ " n_procs = 96,\n",
+ " train_dir = \"/home/jeffrey/Downloads/reddit/per_user_data/train\",\n",
+ " test_dir=\"/home/jeffrey/Downloads/reddit/new_small_data/test\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<torch._C.Generator at 0x7fc21c0c6d30>"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "torch.manual_seed(13)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "id": "hi0N5rB5xBWn"
+ },
+ "outputs": [],
+ "source": [
+ "if torch.cuda.is_available():\n",
+ " device = \"cuda\"\n",
+ "else:\n",
+ " device = \"cpu\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 35
+ },
+ "id": "6lO3uYsmxNYz",
+ "outputId": "b170b610-f21e-465d-fcd6-b7e6989e73e5"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'cpu'"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "device"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## CNN Model Training <a class=\"anchor\" id=\"train\"></a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def _load_vocab(VOCABULARY_PATH):\n",
+ " vocab_file = pickle.load(open(VOCABULARY_PATH, 'rb'))\n",
+ " vocab = collections.defaultdict(lambda: vocab_file['unk_symbol'])\n",
+ " vocab.update(vocab_file['vocab'])\n",
+ "\n",
+ " return vocab, vocab_file['size'], vocab_file['unk_symbol'], vocab_file['pad_symbol']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vocab_path = os.path.join(train_dir, '../../vocab/reddit_vocab.pck')\n",
+ "vocab, vocab_size, unk_symbol, pad_symbol = _load_vocab(vocab_path)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def prepare_data(data):\n",
+ " data_x = data['x']\n",
+ " data_y = data['y']\n",
+ "\n",
+ " # flatten lists\n",
+ " def flatten_lists(data_x_by_comment, data_y_by_comment):\n",
+ " data_x_by_seq, data_y_by_seq = [], []\n",
+ " for c, l in zip(data_x_by_comment, data_y_by_comment):\n",
+ " data_x_by_seq.extend(c)\n",
+ " data_y_by_seq.extend(l['target_tokens'])\n",
+ "\n",
+ " return data_x_by_seq, data_y_by_seq\n",
+ "\n",
+ " data_x, data_y = flatten_lists(data_x, data_y)\n",
+ "\n",
+ " data_x_processed = process_x(data_x)\n",
+ " data_y_processed = process_y(data_y)\n",
+ "\n",
+ " filtered_x, filtered_y = [], []\n",
+ " for i in range(len(data_x_processed)):\n",
+ " if (np.sum(data_y_processed[i]) != 0):\n",
+ " filtered_x.append(data_x_processed[i])\n",
+ " filtered_y.append(data_y_processed[i])\n",
+ "\n",
+ " return (filtered_x, filtered_y)\n",
+ "\n",
+ "def _tokens_to_ids(raw_batch):\n",
+ " def tokens_to_word_ids(tokens, word2id):\n",
+ " return [word2id[word] for word in tokens]\n",
+ "\n",
+ " to_ret = [tokens_to_word_ids(seq, vocab) for seq in raw_batch]\n",
+ " return np.array(to_ret)\n",
+ "\n",
+ "def process_x(raw_x_batch):\n",
+ " \"\"\"\n",
+ " copied from https://gitlab.epfl.ch/sacs/efficient-federated-learning/-/blob/master/grad_guessing/data_utils.py\n",
+ " Parameters\n",
+ " ----------\n",
+ " raw_x_batch\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ "\n",
+ " \"\"\"\n",
+ " tokens = _tokens_to_ids([s for s in raw_x_batch])\n",
+ " return tokens\n",
+ "\n",
+ "def process_y( raw_y_batch):\n",
+ " \"\"\"\n",
+ " copied from https://gitlab.epfl.ch/sacs/efficient-federated-learning/-/blob/master/grad_guessing/data_utils.py\n",
+ " Parameters\n",
+ " ----------\n",
+ " raw_y_batch\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ "\n",
+ " \"\"\"\n",
+ " tokens = _tokens_to_ids([s for s in raw_y_batch])\n",
+ "\n",
+ " def getNextWord(token_ids):\n",
+ " n = len(token_ids)\n",
+ " for i in range(n):\n",
+ " # gets the word at the end of the phrase that should be predicted\n",
+ " # that is the last token that is not a pad.\n",
+ " if (token_ids[n - i - 1] != pad_symbol):\n",
+ " return token_ids[n - i - 1]\n",
+ " return pad_symbol\n",
+ "\n",
+ " return [getNextWord(t) for t in tokens]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "id": "9LpgzEw1s-xo"
+ },
+ "outputs": [],
+ "source": [
+ "def __read_file__(file_path):\n",
+ " with open(file_path, \"r\") as inf:\n",
+ " client_data = json.load(inf)\n",
+ " return (\n",
+ " client_data[\"users\"],\n",
+ " client_data[\"num_samples\"],\n",
+ " client_data[\"user_data\"],\n",
+ " )\n",
+ "\n",
+ "def __read_dir__(data_dir):\n",
+ " users = []\n",
+ " num_samples = []\n",
+ " data = defaultdict(lambda: None)\n",
+ "\n",
+ " files = os.listdir(data_dir)\n",
+ " files = [f for f in files if f.endswith(\".json\")]\n",
+ " for f in files:\n",
+ " file_path = os.path.join(data_dir, f)\n",
+ " u, n, d = __read_file__(file_path)\n",
+ " users.extend(u)\n",
+ " num_samples.extend(n)\n",
+ " data.update(d)\n",
+ " return users, num_samples, data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sizes= None\n",
+ "n_procs = 1 # why can I access n_procs but not train_x in the load_trianset function\n",
+ "train_x = []\n",
+ "train_y = []"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "id": "QBu1kiw8s-xr"
+ },
+ "outputs": [],
+ "source": [
+ "def load_trainset():\n",
+ " files = os.listdir(train_dir)\n",
+ " files = [f for f in files if f.endswith(\".json\")]\n",
+ " files.sort()\n",
+ " c_len = len(files)\n",
+ " sizes = None\n",
+ " print(n_procs)\n",
+ " if sizes == None: # Equal distribution of data among processes\n",
+ " e = c_len // n_procs\n",
+ " frac = e / c_len\n",
+ " sizes = [frac] * n_procs\n",
+ " sizes[-1] += 1.0 - frac * n_procs\n",
+ " uid = 0\n",
+ " my_clients = DataPartitioner(files, sizes).use(0)\n",
+ " my_clients = list(my_clients)\n",
+ " my_train_data = {\"x\": [], \"y\": []}\n",
+ " #self.clients = []\n",
+ " num_samples = []\n",
+ " for i in range(my_clients.__len__()):\n",
+ " cur_file = my_clients.__getitem__(i)\n",
+ "\n",
+ " clients, _, train_data = __read_file__(\n",
+ " os.path.join(train_dir, cur_file)\n",
+ " )\n",
+ " for cur_client in clients:\n",
+ " #self.clients.append(cur_client)\n",
+ " processed_x, processed_y = prepare_data(train_data[cur_client])\n",
+ " # processed_x is an list of fixed size word id arrays that represent a phrase\n",
+ " # processed_y is a list of word ids that each represent the next word of a phrase\n",
+ " my_train_data[\"x\"].extend(processed_x)\n",
+ " my_train_data[\"y\"].extend(processed_y)\n",
+ " num_samples.append(len(processed_y))\n",
+ " # turns the list of lists into a single list\n",
+ " train_y = np.array(my_train_data[\"y\"], dtype=np.dtype(\"int64\")).reshape(-1)\n",
+ " train_x = np.array(my_train_data[\"x\"], dtype=np.dtype(\"int64\"))#.reshape(-1)\n",
+ " print(len(train_x), len(train_y))\n",
+ " print(\"train_x.shape:\", str(train_x.shape))\n",
+ " print(\"train_y.shape:\", str(train_y.shape))\n",
+ " assert train_x.shape[0] == train_y.shape[0]\n",
+ " assert train_x.shape[0] > 0\n",
+ " return train_x, train_y"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_x = []\n",
+ "test_y = []"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def load_testset():\n",
+ " \"\"\"\n",
+ " Loads the testing set.\n",
+ "\n",
+ " \"\"\"\n",
+ " _, _, d = __read_dir__(test_dir)\n",
+ " test_x = []\n",
+ " test_y = []\n",
+ " for test_data in d.values():\n",
+ " processed_x, processed_y = prepare_data(test_data)\n",
+ " # processed_x is an list of fixed size word id arrays that represent a phrase\n",
+ " # processed_y is a list of word ids that each represent the next word of a phrase\n",
+ " test_x.extend(processed_x)\n",
+ " test_y.extend(processed_y)\n",
+ " test_y = np.array(test_y, dtype=np.dtype(\"int64\")).reshape(-1)\n",
+ " test_x = np.array(test_x, dtype=np.dtype(\"int64\"))\n",
+ " print(test_x)\n",
+ " print(len(test_x), len(test_x))\n",
+ " print(\"test_x.shape:\", str(test_x.shape))\n",
+ " print(\"test_y.shape:\", str(test_y.shape))\n",
+ " assert test_x.shape[0] == test_y.shape[0]\n",
+ " assert test_x.shape[0] > 0\n",
+ " return test_x, test_y"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1\n",
+ "70642 70642\n",
+ "train_x.shape: (70642, 10)\n",
+ "train_y.shape: (70642,)\n"
+ ]
+ }
+ ],
+ "source": [
+ "train_x, train_y = load_trainset()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([[ 5, 953, 1341, ..., 834, 298, 1288],\n",
+ " [ 436, 1060, 6, ..., 0, 0, 0],\n",
+ " [ 5, 7948, 1, ..., 7654, 1, 1],\n",
+ " ...,\n",
+ " [ 67, 433, 1465, ..., 0, 0, 0],\n",
+ " [ 5, 13, 119, ..., 12, 17, 13],\n",
+ " [ 324, 324, 324, ..., 0, 0, 0]])"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "train_x"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[[ 5 90 1 ... 6 0 0]\n",
+ " [ 5 13 1121 ... 75 26 110]\n",
+ " [ 27 13 1510 ... 13 4813 21]\n",
+ " ...\n",
+ " [ 5 1784 1 ... 1 1026 4]\n",
+ " [1784 4734 489 ... 1 7 3190]\n",
+ " [ 761 75 1 ... 0 0 0]]\n",
+ "24961 24961\n",
+ "test_x.shape: (24961, 10)\n",
+ "test_y.shape: (24961,)\n"
+ ]
+ }
+ ],
+ "source": [
+ "test_x, test_y = load_testset()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([[ 5, 90, 1, ..., 6, 0, 0],\n",
+ " [ 5, 13, 1121, ..., 75, 26, 110],\n",
+ " [ 27, 13, 1510, ..., 13, 4813, 21],\n",
+ " ...,\n",
+ " [ 5, 1784, 1, ..., 1, 1026, 4],\n",
+ " [1784, 4734, 489, ..., 1, 7, 3190],\n",
+ " [ 761, 75, 1, ..., 0, 0, 0]])"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "test_x"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "id": "mAEASHr2s-x1"
+ },
+ "outputs": [],
+ "source": [
+ "VOCAB_LEN = 9999 \n",
+ "SEQ_LEN = 10\n",
+ "EMBEDDING_DIM = 200"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from decentralizepy.models.Model import Model\n",
+ "import torch.nn.functional as F"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {
+ "id": "GPyZ2C8ws-x9"
+ },
+ "outputs": [],
+ "source": [
+ "class RNN(Model):\n",
+ " \"\"\"\n",
+ " Class for a RNN Model for Reddit\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " def __init__(self):\n",
+ " \"\"\"\n",
+ " Constructor. Instantiates the RNN Model to predict the next word of a sequence of word.\n",
+ " Based on the TensorFlow model found here: https://gitlab.epfl.ch/sacs/efficient-federated-learning/-/blob/master/grad_guessing/data_utils.py\n",
+ " \"\"\"\n",
+ " super().__init__()\n",
+ "\n",
+ " # input_length does not exist\n",
+ " self.embedding = nn.Embedding(VOCAB_LEN, EMBEDDING_DIM, padding_idx=0)\n",
+ " self.rnn_cells = nn.LSTM(EMBEDDING_DIM, 256, batch_first=True, num_layers=2) # not sure about the first argument input_size\n",
+ " # activation function is added in the forward pass\n",
+ " # Note: the tensorflow implementation did not use any activation function in this step?\n",
+ " # should I use one.\n",
+ " self.l1 = nn.Linear(256, 128)\n",
+ " # the tf model used sofmax activation here\n",
+ " self.l2 = nn.Linear(128, VOCAB_LEN)\n",
+ "\n",
+ " def forward(self, x):\n",
+ " \"\"\"\n",
+ " Forward pass of the model\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " x : torch.tensor\n",
+ " The input torch tensor\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " torch.tensor\n",
+ " The output torch tensor\n",
+ "\n",
+ " \"\"\"\n",
+ " x = self.embedding(x)\n",
+ " #print(x.shape)\n",
+ " x = self.rnn_cells(x)\n",
+ " #print(x[0].shape)\n",
+ " #print(x[1][0].shape)\n",
+ " last_layer_output = x[1][0][1,...]\n",
+ " #print(\"last_layer:\", last_layer_output.shape)\n",
+ " x = F.relu(self.l1(x[1][0][1,...]))\n",
+ " #print(x.shape)\n",
+ " x = self.l2(x)\n",
+ " # softmax is applied by the CrossEntropyLoss used during training\n",
+ " return x\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "bCgW8ClBs-x_"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {
+ "id": "oBGwcwZks-yA"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from torch.utils.data import Dataset\n",
+ "\n",
+ "class RedditDataset(Dataset):\n",
+ " def __init__(self, training, transform=None, target_transform=None):\n",
+ " if training:\n",
+ " #with open(train_dir+\"femnist.pkl\", \"rb\") as f:\n",
+ " #train = pickle.load(f)\n",
+ " self.data = train_x\n",
+ " self.label = train_y\n",
+ " else: \n",
+ " #with open(train_dir+\"femnist_test.pkl\", \"rb\") as f:\n",
+ " #test = pickle.load(f)\n",
+ " self.data = test_x\n",
+ " self.label = test_y\n",
+ " self.transform = transform\n",
+ " self.target_transform = target_transform\n",
+ "\n",
+ " def __len__(self):\n",
+ " return len(self.label)\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " return self.data[idx], self.label[idx]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {
+ "id": "U3boC_N4s-yC"
+ },
+ "outputs": [],
+ "source": [
+ "from torch.utils.data import DataLoader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {
+ "id": "sJsrQXkEs-yD"
+ },
+ "outputs": [],
+ "source": [
+ "trainset = RedditDataset(True)\n",
+ "testset = RedditDataset(False)\n",
+ "\n",
+ "train_dataloader = DataLoader(trainset, batch_size=16, shuffle=True)\n",
+ "test_dataloader = DataLoader(testset, batch_size=128, shuffle=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "4416"
+ ]
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(train_dataloader)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "70656"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "552*128"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {
+ "id": "e65Izyv0s-yE"
+ },
+ "outputs": [],
+ "source": [
+ "lr = 0.1\n",
+ "model = RNN().to(device)\n",
+ "loss_fn = nn.CrossEntropyLoss()\n",
+ "optimizer = torch.optim.SGD(model.parameters(), lr=lr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "eqOXilqMs-yF",
+ "outputId": "06799a3b-983b-4f51-a7bd-a901c041bd05"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "loss: 4.976342 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 26.0%, Avg loss: 4.879616 \n",
+ "\n",
+ "loss: 4.882441 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 28.8%, Avg loss: 4.710636 \n",
+ "\n",
+ "loss: 4.665762 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 30.0%, Avg loss: 4.597566 \n",
+ "\n",
+ "loss: 4.593148 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 31.8%, Avg loss: 4.533113 \n",
+ "\n",
+ "loss: 4.442960 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 32.2%, Avg loss: 4.463360 \n",
+ "\n",
+ "loss: 4.288299 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 30.8%, Avg loss: 4.559918 \n",
+ "\n",
+ "loss: 4.191413 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 30.7%, Avg loss: 4.568339 \n",
+ "\n",
+ "loss: 3.979227 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 31.4%, Avg loss: 4.590676 \n",
+ "\n",
+ "loss: 3.857259 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 30.8%, Avg loss: 4.667198 \n",
+ "\n",
+ "loss: 3.840112 [ 8832/70642]\n",
+ "epoch:\n",
+ "Test Error: \n",
+ " Accuracy: 30.9%, Avg loss: 4.786180 \n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "stats = {\"train\": [], \"test\":[]}\n",
+ "loss_mvg = None\n",
+ "for e in range(10):\n",
+ " #training\n",
+ " batch = 0\n",
+ " for X, y in train_dataloader:\n",
+ " #print(\"grad: \"+str(model.conv1.bias.grad))\n",
+ " #old = model.conv1.bias.clone()\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " #print(len(label))\n",
+ " size = len(train_dataloader.dataset)\n",
+ " model.train()\n",
+ " # Compute prediction error\n",
+ " pred = model(X)\n",
+ " #print(X.shape, y.shape) is torch.Size([128, 10]) torch.Size([128])\n",
+ " loss = loss_fn(pred, y)\n",
+ " # Backpropagation\n",
+ " loss.backward()\n",
+ " #print(\"grad2: \"+str(model.conv1.bias.grad))\n",
+ " optimizer.step()\n",
+ " model.zero_grad()\n",
+ " #optimizer.zero_grad()\n",
+ " \n",
+ " #resetting the optimizer\n",
+ " # optimizer.load_state_dict(model.state_dict())\n",
+ " vals = optimizer.state.values()\n",
+ " #print(optimizer.state.values())\n",
+ " if not loss_mvg:\n",
+ " loss_mvg = loss.item()\n",
+ " else:\n",
+ " loss_mvg = 0.99*loss_mvg + 0.01*loss.item()\n",
+ " \n",
+ " batch += 1\n",
+ " \n",
+ " loss, current = loss.item(), batch * len(X)\n",
+ " print(f\"loss: {loss_mvg:>7f} [{current:>5d}/{size:>5d}]\")\n",
+ " stats[\"train\"].append([batch, e*size + current, loss])\n",
+ "\n",
+ "\n",
+ " #testing\n",
+ " size = len(test_dataloader.dataset)\n",
+ " num_batches = len(test_dataloader)\n",
+ " model.eval()\n",
+ " test_loss, correct = 0, 0\n",
+ " with torch.no_grad():\n",
+ " for X, y in test_dataloader:\n",
+ " X = X.to(device)\n",
+ " y = y.to(device)\n",
+ " pred = model(X)\n",
+ " test_loss += loss_fn(pred, y).item()\n",
+ " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
+ " test_loss /= num_batches\n",
+ " correct /= size\n",
+ " print(\"epoch:\")\n",
+ " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
+ " stats[\"test\"].append([e, test_loss, 100*correct])\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "128 0.001: Accuracy: 32.3%, Avg loss: 4.716327 \n",
+ "64 0.001: Accuracy: 32.6%, Avg loss: 4.723278 "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "4P-VA0vcs-yH"
+ },
+ "outputs": [],
+ "source": [
+ "with open(train_dir+\"/results:128:\"+str(lr)+\".pkl\", \"wb\") as f:\n",
+ " pickle.dump(stats, f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "641-b_VCvT2b",
+ "outputId": "cced38ab-5c04-45b2-faf4-e73327126159"
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "F_OKqiiHs-yJ",
+ "outputId": "65786b88-05f4-42fa-a851-03397ef4457a"
+ },
+ "outputs": [],
+ "source": [
+ "lrs = [0.1, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00001]\n",
+ "for l in lrs:\n",
+ " with open(train_dir+\"/results:128:\"+str(l)+\".pkl\", \"rb\") as f:\n",
+ " res = pickle.load(f)\n",
+ " print(str(l)+\": \" + str(np.amax(res[\"test\"], axis=0)))#+ str(np.max(res[\"test\"]))\n",
+ " # print(str(l)+\": \" + str(res[\"test\"]))\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "rADw-XkfKjOo",
+ "outputId": "06c54a2c-f7c2-4610-f879-3e1c2f98543f"
+ },
+ "outputs": [],
+ "source": [
+ "lrs = [0.1, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00001]\n",
+ "for l in lrs:\n",
+ " with open(train_dir+\"/results:128:\"+str(l)+\".pkl\", \"rb\") as f:\n",
+ " res = pickle.load(f)\n",
+ " # print(str(l)+\": \" + str(np.amax(res[\"test\"], axis=0)))#+ str(np.max(res[\"test\"]))\n",
+ " print(str(l)+\": \" + str(res[\"test\"]))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "HGpNYzG_s-yJ",
+ "outputId": "783622a5-249f-4dd8-d242-fc6dfa47443c"
+ },
+ "outputs": [],
+ "source": [
+ "import torch\n",
+ "resnet = torch.hub.load('pytorch/vision:v0.10.0', 'resnet18', pretrained=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "uZFgT6wss-yL",
+ "outputId": "10f8fc51-abb7-4c2b-f608-85229f3de29d",
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "total = 0\n",
+ "for i in resnet.state_dict().values():\n",
+ " total += i.flatten().size(dim=0)\n",
+ "print(total)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Optimizer analysis <a class=\"anchor\" id=\"optim\"></a>"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "name": "learningrate.ipynb",
+ "provenance": []
+ },
+ "interpreter": {
+ "hash": "996934296aa9d79be6c3d800a38d8fdb7dfa8fe7bb07df178f1397cde2cb8742"
+ },
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/random files/test_elias.py b/random files/test_elias.py
new file mode 100644
index 0000000000000000000000000000000000000000..10366741cc8078665972c00c8eaf86d17ed81816
--- /dev/null
+++ b/random files/test_elias.py
@@ -0,0 +1,177 @@
+import numpy as np
+import time
+import os, random, pickle, lzma, leb128, lz4.frame
+import uvarint
+
+# arr = np.random.poisson(3, 500000) #np.random.randint(1, 20, 500000, dtype=np.int32)
+# arr=arr.astype(np.int32)
+# print("got the array")
+# t5 = time.time()
+#
+#
+# lzfour= lz4.frame.compress(arr)
+#
+# p = pickle.dumps(lzfour)
+# print(len(p), len(lzfour))
+# t6 = time.time()
+#
+# p = pickle.dumps(p)
+#
+# t7 = time.time()
+# print(t6 -t5, t7 -t5)
+#
+# # elias implementation: taken from this stack overflow post:
+# # https://stackoverflow.com/questions/62843156/python-fast-compression-of-large-amount-of-numbers-with-elias-gamma
+# def encode(a):
+# a = a.view(f'u{a.itemsize}')
+# l = np.log2(a).astype('u1')
+# L = ((l<<1)+1).cumsum()
+# print("L,", L, len(L))
+# out = np.zeros(L[-1] + np.array([64], dtype = "u1")[0], 'u1')
+# print("lmax", l.max()+1)
+# for i in range(l.max()+1):
+# out[L-i-1] += (a>>i)&1
+# print("out:", out, out[-10:])
+# s = np.array([out.size], dtype=np.int64)
+#
+# print(s)
+# size = np.ndarray(8, dtype='u1', buffer=s.data)
+# print("size:", size)
+# # out[-8:] = size
+# ss = s[0]
+# # for i in range(64):
+# # out[int(L[-1] + 63 - i)] += (ss>>i)&1
+# print("out", out, out[-68:])
+#
+# # out.data.contiguous
+# packed = np.packbits(out)
+# packed[-8:] = size
+# print("packed:", packed[-10:])
+# print("out", out, out[-68:])
+# return packed, out.size
+#
+# def decode(b,n):
+# print(b,)
+# n_arr = b[-8:]
+# print("n_arr:", n_arr)
+# n = np.ndarray(1, dtype=np.int64, buffer=n_arr.data)[0]
+# print("n:", n)
+# b = b[:-8]
+# b = np.unpackbits(b,count=n).view(bool)
+# s = b.nonzero()[0]
+# s = (s<<1).repeat(np.diff(s,prepend=-1))
+# s -= np.arange(-1,len(s)-1)
+# s = s.tolist() # list has faster __getitem__
+# ns = len(s)
+# def gen():
+# idx = 0
+# yield idx
+# while idx < ns:
+# idx = s[idx]
+# yield idx
+# offs = np.fromiter(gen(),int)
+# sz = np.diff(offs)>>1
+# mx = sz.max()+1
+# out = np.zeros(offs.size-1,int)
+# for i in range(mx):
+# out[b[offs[1:]-i-1] & (sz>=i)] += 1<<i
+# return out
+#
+# arr = np.random.poisson(3, 500000) + 1 # elias does not work on 0s # .frame.decompress
+# arr= arr.astype(np.int64)
+# print("arr:", arr)
+#
+#
+# # t0 = time.time()
+# # p = pickle.dumps(arr)
+# # t1 = time.time()
+# #
+# # z = lzma.compress(arr)
+# #
+# # t2 = time.time()
+# #
+# # # From https://stackoverflow.com/questions/68968796/variable-length-integer-encoding
+# # leb = b''.join(map(leb128.LEB128U.encode, arr))
+# # t3 = time.time()
+# #
+# # uvar = b''.join(map(uvarint.encode, arr))
+# t4 = time.time()
+#
+# elias, n = encode(arr)
+#
+# t5 = time.time()
+# #
+# # lzfour= lz4.frame.compress(arr.tobytes("C"))
+# #
+# # t6 = time.time()
+#
+# # print(elias)
+# # print(n)
+# # print(elias.size)
+# # print(elias.itemsize)
+# # print("array size'd:", f'{ arr.size * arr.itemsize:,}')
+# # print("pickle'd:", f'{len(p):,}')
+# # print("lzma'd:", f'{len(z):,}')
+# # print("leb128'd:", f'{len(leb):,}')
+# # print("uvarint'd:", f'{len(leb):,}')
+# # print("elias'd:", f'{elias.size:,}')
+# # print("elias'd:", f'{len(elias):,}')
+# # print("elias'd:", f'{len(pickle.dumps(elias)):,}')
+# # print("lz4'd:", f'{len(lzfour):,}')
+# # print(f"pickle: {t1-t0:.5f}s, lzma: {t2-t1:.5f}s, leb128 {t3-t2:.5f}s, uvarint: {t4-t3:.5f}s, elias: {t5-t4:.5f}s, lz4: {t6-t5:.5f}s")
+#
+#
+#
+# # decode
+# d0 = time.time()
+#
+# arr_dec = decode(elias, n)
+# print(arr.dtype, arr_dec.dtype)
+# print(len(arr) , len(arr_dec))
+# assert len(arr) == len(arr_dec)
+# assert (arr == arr_dec).all()
+#
+# d1 = time.time()
+#
+# # arr_dec = lz4.frame.decompress(lzfour)
+# # arr_dec = np.frombuffer(arr_dec, dtype=np.int32)
+# d2 = time.time()
+# # print(arr_dec)
+#
+# print(d1 -d0, d2 - d1)
+
+
+from decentralizepy.compression.Elias import Elias
+
+arr = np.random.poisson(30, 50000) + 1 # elias does not work on 0s # .frame.decompress
+arr= arr.astype(np.int32)
+arr = np.cumsum(arr).astype(np.int32)
+print("arr:", arr[:3], arr[-3:])
+
+#print("Elias of framework:", arr, np.diff(arr), len(np.diff(arr)))
+eliaso = Elias()
+t1 = time.time()
+comp = eliaso.compress(arr)
+t2 = time.time()
+print("size:", comp.dtype, len(comp), len(pickle.dumps(arr)), len(pickle.dumps(comp)))
+arr_dec = eliaso.decompress(comp)
+t3= time.time()
+print(t2-t1, t3-t2, t3-t1)
+print("arr_dec:", arr_dec[:3], arr_dec[-3:])
+print(arr.dtype, arr_dec.dtype)
+print(len(arr) , len(arr_dec))
+assert len(arr) == len(arr_dec)
+assert (arr == arr_dec).all()
+
+
+arr = np.array([0,1,2,8])
+
+comp = eliaso.compress(arr)
+arr_dec = eliaso.decompress(comp)
+print("test for fix", arr, arr_dec)
+print(arr_dec)
+print(arr.dtype, arr_dec.dtype)
+print(len(arr) , len(arr_dec))
+assert len(arr) == len(arr_dec)
+assert (arr == arr_dec).all()
+
diff --git a/random files/testing_time_offset_dup.py b/random files/testing_time_offset_dup.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f85c323acd09410bcedcec8cc3d45e313e44dc7
--- /dev/null
+++ b/random files/testing_time_offset_dup.py
@@ -0,0 +1,25 @@
+import time
+
+
+print(time.time())
+
+print(int(time.time()))
+
+intervall = 60*5 # NEW Batch of ports avter 5 minutes
+
+seconds = int(time.time()) // intervall
+
+
+
+# we reset after 200 mins
+
+port_offset_factor = seconds % 40
+
+in_day = 40
+ports = 400
+breath = in_day * ports
+
+offset = port_offset_factor * ports
+print(intervall, seconds, port_offset_factor, offset, seconds % in_day, breath)
+
+# There are 65535 ports in tcp
diff --git a/random files/top10_Histogram.png b/random files/top10_Histogram.png
new file mode 100644
index 0000000000000000000000000000000000000000..a18d55c04ffaa7dea45dedb550930a1ab38eb57b
Binary files /dev/null and b/random files/top10_Histogram.png differ
diff --git a/random files/wavelet_bad.png b/random files/wavelet_bad.png
new file mode 100644
index 0000000000000000000000000000000000000000..91693c68727f7e096793307f3be0a1a7af87675f
Binary files /dev/null and b/random files/wavelet_bad.png differ
diff --git a/random files/wavelet_bad_approx.png b/random files/wavelet_bad_approx.png
new file mode 100644
index 0000000000000000000000000000000000000000..093529b6ead27ab067bc452ee474f286c1e27ef2
Binary files /dev/null and b/random files/wavelet_bad_approx.png differ
diff --git a/random files/wavelet_bad_approx_good.png b/random files/wavelet_bad_approx_good.png
new file mode 100644
index 0000000000000000000000000000000000000000..408957a1593494209dafcd7e8d6232de5a52fbba
Binary files /dev/null and b/random files/wavelet_bad_approx_good.png differ
diff --git a/random files/wavelet_bad_approx_partial.png b/random files/wavelet_bad_approx_partial.png
new file mode 100644
index 0000000000000000000000000000000000000000..d2791df7ccdbbdc3214ca9969d8ec99754559010
Binary files /dev/null and b/random files/wavelet_bad_approx_partial.png differ
diff --git a/random files/wavelet_good.png b/random files/wavelet_good.png
new file mode 100644
index 0000000000000000000000000000000000000000..408957a1593494209dafcd7e8d6232de5a52fbba
Binary files /dev/null and b/random files/wavelet_good.png differ
diff --git a/src/__init__.py b/src/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/src/decentralizepy/compression/__init__.py b/src/decentralizepy/compression/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391