diff --git a/examples/agp-pruning/resnet50.schedule_agp.filters_2.yaml b/examples/agp-pruning/resnet50.schedule_agp.filters_2.yaml new file mode 100755 index 0000000000000000000000000000000000000000..e5713192a62a37ad580bb0ee96c4045341435fe9 --- /dev/null +++ b/examples/agp-pruning/resnet50.schedule_agp.filters_2.yaml @@ -0,0 +1,192 @@ +# +# This schedule performs filter-pruning using L1-norm ranking and AGP for the setting the pruning-rate decay. +# +# Best Top1: 74.782 (epoch 94) +# No. of Parameters: 12,671,168 (of 25,502,912) = 49.69% dense (50.31% sparse) +# Total MACs: 2,037,186,560 (of 4,089,184,256) = 49.82% compute = 2.01x +# +# time python3 compress_classifier.py -a=resnet50 --pretrained -p=50 ../../../data.imagenet/ -j=22 --epochs=100 --lr=0.0005 --compress=resnet50.schedule_agp.filters_2.yaml --validation-size=0 --num-best-scores=10 +# +# Parameters: +# +----+-------------------------------------+--------------------+---------------+----------------+------------+------------+----------+----------+----------+------------+---------+----------+------------+ +# | | Name | Shape | NNZ (dense) | NNZ (sparse) | Cols (%) | Rows (%) | Ch (%) | 2D (%) | 3D (%) | Fine (%) | Std | Mean | Abs-Mean | +# |----+-------------------------------------+--------------------+---------------+----------------+------------+------------+----------+----------+----------+------------+---------+----------+------------| +# | 0 | module.conv1.weight | (64, 3, 7, 7) | 9408 | 9408 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.11098 | -0.00043 | 0.06774 | +# | 1 | module.layer1.0.conv1.weight | (32, 64, 1, 1) | 2048 | 2048 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.07631 | -0.00587 | 0.04636 | +# | 2 | module.layer1.0.conv2.weight | (32, 32, 3, 3) | 9216 | 9216 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.04019 | 0.00147 | 0.02596 | +# | 3 | module.layer1.0.conv3.weight | (256, 32, 1, 1) | 8192 | 8192 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03788 | -0.00045 | 0.02391 | +# | 4 | module.layer1.0.downsample.0.weight | (256, 64, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.05137 | -0.00304 | 0.02857 | +# | 5 | module.layer1.1.conv1.weight | (32, 256, 1, 1) | 8192 | 8192 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03148 | 0.00120 | 0.02169 | +# | 6 | module.layer1.1.conv2.weight | (32, 32, 3, 3) | 9216 | 9216 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03669 | 0.00017 | 0.02582 | +# | 7 | module.layer1.1.conv3.weight | (256, 32, 1, 1) | 8192 | 8192 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03162 | -0.00060 | 0.02006 | +# | 8 | module.layer1.2.conv1.weight | (32, 256, 1, 1) | 8192 | 8192 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02993 | 0.00020 | 0.02192 | +# | 9 | module.layer1.2.conv2.weight | (32, 32, 3, 3) | 9216 | 9216 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03611 | 0.00009 | 0.02719 | +# | 10 | module.layer1.2.conv3.weight | (256, 32, 1, 1) | 8192 | 8192 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02778 | -0.00228 | 0.01659 | +# | 11 | module.layer2.0.conv1.weight | (128, 256, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03164 | -0.00144 | 0.02232 | +# | 12 | module.layer2.0.conv2.weight | (64, 128, 3, 3) | 73728 | 73728 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02147 | 0.00000 | 0.01595 | +# | 13 | module.layer2.0.conv3.weight | (512, 64, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02703 | 0.00005 | 0.01656 | +# | 14 | module.layer2.0.downsample.0.weight | (512, 256, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02051 | -0.00038 | 0.01206 | +# | 15 | module.layer2.1.conv1.weight | (64, 512, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01744 | -0.00008 | 0.01081 | +# | 16 | module.layer2.1.conv2.weight | (128, 64, 3, 3) | 73728 | 73728 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02022 | 0.00011 | 0.01301 | +# | 17 | module.layer2.1.conv3.weight | (512, 128, 1, 1) | 65536 | 65536 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01982 | -0.00107 | 0.01153 | +# | 18 | module.layer2.2.conv1.weight | (64, 512, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02328 | -0.00053 | 0.01618 | +# | 19 | module.layer2.2.conv2.weight | (64, 64, 3, 3) | 36864 | 36864 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02380 | 0.00012 | 0.01667 | +# | 20 | module.layer2.2.conv3.weight | (512, 64, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02561 | 0.00015 | 0.01784 | +# | 21 | module.layer2.3.conv1.weight | (64, 512, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02327 | -0.00090 | 0.01733 | +# | 22 | module.layer2.3.conv2.weight | (64, 64, 3, 3) | 36864 | 36864 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02368 | -0.00043 | 0.01789 | +# | 23 | module.layer2.3.conv3.weight | (512, 64, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02287 | -0.00116 | 0.01577 | +# | 24 | module.layer3.0.conv1.weight | (256, 512, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02737 | -0.00126 | 0.01964 | +# | 25 | module.layer3.0.conv2.weight | (128, 256, 3, 3) | 294912 | 294912 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01679 | -0.00019 | 0.01241 | +# | 26 | module.layer3.0.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02290 | -0.00043 | 0.01647 | +# | 27 | module.layer3.0.downsample.0.weight | (1024, 512, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01431 | -0.00000 | 0.00982 | +# | 28 | module.layer3.1.conv1.weight | (128, 1024, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01517 | -0.00037 | 0.01072 | +# | 29 | module.layer3.1.conv2.weight | (128, 128, 3, 3) | 147456 | 147456 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01683 | -0.00006 | 0.01212 | +# | 30 | module.layer3.1.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01959 | -0.00063 | 0.01394 | +# | 31 | module.layer3.2.conv1.weight | (128, 1024, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01547 | -0.00032 | 0.01103 | +# | 32 | module.layer3.2.conv2.weight | (128, 128, 3, 3) | 147456 | 147456 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01644 | -0.00056 | 0.01214 | +# | 33 | module.layer3.2.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01832 | -0.00054 | 0.01331 | +# | 34 | module.layer3.3.conv1.weight | (128, 1024, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01675 | -0.00058 | 0.01250 | +# | 35 | module.layer3.3.conv2.weight | (128, 128, 3, 3) | 147456 | 147456 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01552 | -0.00053 | 0.01179 | +# | 36 | module.layer3.3.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01741 | -0.00095 | 0.01280 | +# | 37 | module.layer3.4.conv1.weight | (128, 1024, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01738 | -0.00080 | 0.01312 | +# | 38 | module.layer3.4.conv2.weight | (128, 128, 3, 3) | 147456 | 147456 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01539 | -0.00064 | 0.01169 | +# | 39 | module.layer3.4.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01709 | -0.00126 | 0.01253 | +# | 40 | module.layer3.5.conv1.weight | (128, 1024, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01868 | -0.00072 | 0.01434 | +# | 41 | module.layer3.5.conv2.weight | (128, 128, 3, 3) | 147456 | 147456 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01528 | -0.00073 | 0.01170 | +# | 42 | module.layer3.5.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01853 | -0.00212 | 0.01393 | +# | 43 | module.layer4.0.conv1.weight | (256, 1024, 1, 1) | 262144 | 262144 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02219 | -0.00087 | 0.01715 | +# | 44 | module.layer4.0.conv2.weight | (256, 256, 3, 3) | 589824 | 589824 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01234 | -0.00011 | 0.00962 | +# | 45 | module.layer4.0.conv3.weight | (2048, 256, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01454 | -0.00058 | 0.01133 | +# | 46 | module.layer4.0.downsample.0.weight | (2048, 1024, 1, 1) | 2097152 | 2097152 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00905 | -0.00018 | 0.00689 | +# | 47 | module.layer4.1.conv1.weight | (256, 2048, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01431 | -0.00032 | 0.01119 | +# | 48 | module.layer4.1.conv2.weight | (256, 256, 3, 3) | 589824 | 589824 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01231 | -0.00060 | 0.00965 | +# | 49 | module.layer4.1.conv3.weight | (2048, 256, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01433 | 0.00003 | 0.01110 | +# | 50 | module.layer4.2.conv1.weight | (256, 2048, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01778 | -0.00008 | 0.01397 | +# | 51 | module.layer4.2.conv2.weight | (256, 256, 3, 3) | 589824 | 589824 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01080 | -0.00034 | 0.00850 | +# | 52 | module.layer4.2.conv3.weight | (2048, 256, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01315 | 0.00019 | 0.00992 | +# | 53 | module.fc.weight | (1000, 2048) | 2048000 | 2048000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03325 | 0.00000 | 0.02289 | +# | 54 | Total sparsity: | - | 12671168 | 12671168 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | +# +----+-------------------------------------+--------------------+---------------+----------------+------------+------------+----------+----------+----------+------------+---------+----------+------------+ +# 2018-12-09 13:27:25,875 - Total sparsity: 0.00 +# +# 2018-12-09 13:27:25,875 - --- validate (epoch=99)----------- +# 2018-12-09 13:27:25,875 - 50000 samples (256 per mini-batch) +# 2018-12-09 13:27:46,138 - Epoch: [99][ 50/ 195] Loss 0.728680 Top1 80.640625 Top5 95.507812 +# 2018-12-09 13:27:53,943 - Epoch: [99][ 100/ 195] Loss 0.850403 Top1 78.128906 Top5 94.128906 +# 2018-12-09 13:28:03,180 - Epoch: [99][ 150/ 195] Loss 0.973435 Top1 75.731771 Top5 92.619792 +# 2018-12-09 13:28:10,151 - ==> Top1: 74.738 Top5: 92.080 Loss: 1.018 +# +# 2018-12-09 13:28:10,230 - ==> Best Top1: 75.896 on Epoch: 0 +# 2018-12-09 13:28:10,230 - ==> Best Top1: 75.402 on Epoch: 1 +# 2018-12-09 13:28:10,230 - ==> Best Top1: 74.916 on Epoch: 2 +# 2018-12-09 13:28:10,230 - ==> Best Top1: 74.782 on Epoch: 94 <========== +# 2018-12-09 13:28:10,230 - ==> Best Top1: 74.776 on Epoch: 93 +# 2018-12-09 13:28:10,230 - ==> Best Top1: 74.774 on Epoch: 84 +# 2018-12-09 13:28:10,230 - ==> Best Top1: 74.772 on Epoch: 97 +# 2018-12-09 13:28:10,231 - ==> Best Top1: 74.770 on Epoch: 98 +# 2018-12-09 13:28:10,231 - ==> Best Top1: 74.738 on Epoch: 99 +# 2018-12-09 13:28:10,231 - ==> Best Top1: 74.726 on Epoch: 91 +# 2018-12-09 13:28:10,231 - Saving checkpoint to: logs/resnet50_filters_v3.1___2018.12.07-154945/resnet50_filters_v3.1_checkpoint.pth.tar +# 2018-12-09 13:28:10,458 - --- test --------------------- +# 2018-12-09 13:28:10,458 - 50000 samples (256 per mini-batch) +# 2018-12-09 13:28:30,687 - Test: [ 50/ 195] Loss 0.728680 Top1 80.640625 Top5 95.507812 +# 2018-12-09 13:28:38,854 - Test: [ 100/ 195] Loss 0.850403 Top1 78.128906 Top5 94.128906 +# 2018-12-09 13:28:47,691 - Test: [ 150/ 195] Loss 0.973435 Top1 75.731771 Top5 92.619792 +# 2018-12-09 13:28:54,669 - ==> Top1: 74.738 Top5: 92.080 Loss: 1.018 + +version: 1 + +pruners: + fc_pruner: + class: AutomatedGradualPruner + initial_sparsity : 0.05 + final_sparsity: 0.87 + weights: module.fc.weight + + filter_pruner: + class: L1RankedStructureParameterPruner_AGP + initial_sparsity : 0.05 + final_sparsity: 0.50 + group_type: Filters + weights: [module.layer1.0.conv1.weight, + module.layer1.1.conv1.weight, + module.layer1.2.conv1.weight, + #module.layer2.0.conv1.weight, + module.layer2.1.conv1.weight, + module.layer2.2.conv1.weight, + module.layer2.3.conv1.weight, + #module.layer3.0.conv1.weight, + module.layer3.1.conv1.weight, + module.layer3.2.conv1.weight, + module.layer3.3.conv1.weight, + module.layer3.4.conv1.weight, + module.layer3.5.conv1.weight, + module.layer4.0.conv1.weight, + module.layer4.1.conv1.weight, + module.layer4.2.conv1.weight, + + + module.layer1.0.conv2.weight, + module.layer1.1.conv2.weight, + module.layer1.2.conv2.weight, + module.layer2.0.conv2.weight, + #module.layer2.1.conv2.weight, + module.layer2.2.conv2.weight, + module.layer2.3.conv2.weight, + module.layer3.0.conv2.weight, + module.layer3.1.conv2.weight, + module.layer3.2.conv2.weight, + module.layer3.3.conv2.weight, + module.layer3.4.conv2.weight, + module.layer3.5.conv2.weight, + module.layer4.0.conv2.weight, + module.layer4.1.conv2.weight, + module.layer4.2.conv2.weight] + + fine_pruner: + class: AutomatedGradualPruner + initial_sparsity : 0.05 + final_sparsity: 0.70 + weights: [ + module.layer4.0.conv2.weight, + module.layer4.0.conv3.weight, + module.layer4.0.downsample.0.weight, + module.layer4.1.conv1.weight, + module.layer4.1.conv2.weight, + module.layer4.1.conv3.weight, + module.layer4.2.conv1.weight, + module.layer4.2.conv2.weight, + module.layer4.2.conv3.weight] + +extensions: + net_thinner: + class: 'FilterRemover' + thinning_func_str: remove_filters + arch: 'resnet50' + dataset: 'imagenet' + +lr_schedulers: + pruning_lr: + class: ExponentialLR + gamma: 0.95 + +policies: + - pruner: + instance_name : filter_pruner +# args: +# mini_batch_pruning_frequency: 1 + starting_epoch: 0 + ending_epoch: 30 + frequency: 1 + +# After completeing the pruning, we perform network thinning and continue fine-tuning. + - extension: + instance_name: net_thinner + epochs: [31] + + + - lr_scheduler: + instance_name: pruning_lr + starting_epoch: 40 + ending_epoch: 80 + frequency: 1 diff --git a/examples/agp-pruning/resnet50.schedule_agp.filters_3.yaml b/examples/agp-pruning/resnet50.schedule_agp.filters_3.yaml new file mode 100755 index 0000000000000000000000000000000000000000..d68c3fc1b3770a43d0282ecaa92a8bab04e482f2 --- /dev/null +++ b/examples/agp-pruning/resnet50.schedule_agp.filters_3.yaml @@ -0,0 +1,171 @@ + +# This schedule performs filter-pruning using L1-norm ranking and AGP for the setting the pruning-rate decay. +# +# Best Top1: 75.748 (epoch 94) +# No. of Parameters: 17,329,344 (of 25,502,912) = 67.95% dense (32.05% sparse) +# Total MACs: 2,753,298,432 (of 4,089,184,256) = 67.33% compute = 1.49x +# +# time python3 compress_classifier.py -a=resnet50 --pretrained -p=50 ../../../data.imagenet/ -j=22 --epochs=100 --lr=0.0005 --compress=resnet50.schedule_agp.filters_3.yaml --validation-size=0 --num-best-scores=10 +# +# Parameters: +# +----+-------------------------------------+--------------------+---------------+----------------+------------+------------+----------+----------+----------+------------+---------+----------+------------+ +# | | Name | Shape | NNZ (dense) | NNZ (sparse) | Cols (%) | Rows (%) | Ch (%) | 2D (%) | 3D (%) | Fine (%) | Std | Mean | Abs-Mean | +# |----+-------------------------------------+--------------------+---------------+----------------+------------+------------+----------+----------+----------+------------+---------+----------+------------| +# | 0 | module.conv1.weight | (64, 3, 7, 7) | 9408 | 9408 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.11053 | -0.00040 | 0.06769 | +# | 1 | module.layer1.0.conv1.weight | (64, 64, 1, 1) | 4096 | 4096 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.06357 | -0.00404 | 0.03573 | +# | 2 | module.layer1.0.conv2.weight | (32, 64, 3, 3) | 18432 | 18432 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03310 | 0.00093 | 0.02084 | +# | 3 | module.layer1.0.conv3.weight | (256, 32, 1, 1) | 8192 | 8192 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03770 | -0.00022 | 0.02367 | +# | 4 | module.layer1.0.downsample.0.weight | (256, 64, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.05107 | -0.00305 | 0.02849 | +# | 5 | module.layer1.1.conv1.weight | (64, 256, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02716 | 0.00097 | 0.01802 | +# | 6 | module.layer1.1.conv2.weight | (32, 64, 3, 3) | 18432 | 18432 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03056 | 0.00020 | 0.02092 | +# | 7 | module.layer1.1.conv3.weight | (256, 32, 1, 1) | 8192 | 8192 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03139 | -0.00050 | 0.01988 | +# | 8 | module.layer1.2.conv1.weight | (64, 256, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02660 | 0.00006 | 0.01926 | +# | 9 | module.layer1.2.conv2.weight | (32, 64, 3, 3) | 18432 | 18432 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03028 | -0.00037 | 0.02278 | +# | 10 | module.layer1.2.conv3.weight | (256, 32, 1, 1) | 8192 | 8192 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02762 | -0.00230 | 0.01640 | +# | 11 | module.layer2.0.conv1.weight | (128, 256, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03149 | -0.00137 | 0.02218 | +# | 12 | module.layer2.0.conv2.weight | (64, 128, 3, 3) | 73728 | 73728 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02133 | 0.00000 | 0.01584 | +# | 13 | module.layer2.0.conv3.weight | (512, 64, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02686 | 0.00009 | 0.01642 | +# | 14 | module.layer2.0.downsample.0.weight | (512, 256, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02047 | -0.00043 | 0.01202 | +# | 15 | module.layer2.1.conv1.weight | (128, 512, 1, 1) | 65536 | 65536 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01479 | -0.00015 | 0.00897 | +# | 16 | module.layer2.1.conv2.weight | (64, 128, 3, 3) | 73728 | 73728 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02168 | 0.00056 | 0.01426 | +# | 17 | module.layer2.1.conv3.weight | (512, 64, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02224 | -0.00137 | 0.01297 | +# | 18 | module.layer2.2.conv1.weight | (128, 512, 1, 1) | 65536 | 65536 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02067 | -0.00070 | 0.01441 | +# | 19 | module.layer2.2.conv2.weight | (64, 128, 3, 3) | 73728 | 73728 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02121 | -0.00012 | 0.01501 | +# | 20 | module.layer2.2.conv3.weight | (512, 64, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02533 | 0.00031 | 0.01765 | +# | 21 | module.layer2.3.conv1.weight | (128, 512, 1, 1) | 65536 | 65536 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02153 | -0.00086 | 0.01597 | +# | 22 | module.layer2.3.conv2.weight | (64, 128, 3, 3) | 73728 | 73728 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02162 | -0.00050 | 0.01635 | +# | 23 | module.layer2.3.conv3.weight | (512, 64, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02281 | -0.00109 | 0.01573 | +# | 24 | module.layer3.0.conv1.weight | (256, 512, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02727 | -0.00112 | 0.01952 | +# | 25 | module.layer3.0.conv2.weight | (128, 256, 3, 3) | 294912 | 294912 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01670 | -0.00017 | 0.01233 | +# | 26 | module.layer3.0.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02275 | -0.00041 | 0.01634 | +# | 27 | module.layer3.0.downsample.0.weight | (1024, 512, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01429 | 0.00004 | 0.00978 | +# | 28 | module.layer3.1.conv1.weight | (256, 1024, 1, 1) | 262144 | 262144 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01355 | -0.00048 | 0.00953 | +# | 29 | module.layer3.1.conv2.weight | (128, 256, 3, 3) | 294912 | 294912 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01497 | -0.00012 | 0.01089 | +# | 30 | module.layer3.1.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01943 | -0.00062 | 0.01378 | +# | 31 | module.layer3.2.conv1.weight | (256, 1024, 1, 1) | 262144 | 262144 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01404 | -0.00045 | 0.01007 | +# | 32 | module.layer3.2.conv2.weight | (128, 256, 3, 3) | 294912 | 294912 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01462 | -0.00055 | 0.01092 | +# | 33 | module.layer3.2.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01819 | -0.00049 | 0.01321 | +# | 34 | module.layer3.3.conv1.weight | (256, 1024, 1, 1) | 262144 | 262144 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01545 | -0.00069 | 0.01146 | +# | 35 | module.layer3.3.conv2.weight | (128, 256, 3, 3) | 294912 | 294912 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01450 | -0.00058 | 0.01108 | +# | 36 | module.layer3.3.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01730 | -0.00090 | 0.01271 | +# | 37 | module.layer3.4.conv1.weight | (256, 1024, 1, 1) | 262144 | 262144 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01610 | -0.00086 | 0.01212 | +# | 38 | module.layer3.4.conv2.weight | (128, 256, 3, 3) | 294912 | 294912 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01435 | -0.00078 | 0.01100 | +# | 39 | module.layer3.4.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01690 | -0.00115 | 0.01236 | +# | 40 | module.layer3.5.conv1.weight | (256, 1024, 1, 1) | 262144 | 262144 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01752 | -0.00083 | 0.01335 | +# | 41 | module.layer3.5.conv2.weight | (128, 256, 3, 3) | 294912 | 294912 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01461 | -0.00076 | 0.01118 | +# | 42 | module.layer3.5.conv3.weight | (1024, 128, 1, 1) | 131072 | 131072 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01839 | -0.00203 | 0.01380 | +# | 43 | module.layer4.0.conv1.weight | (512, 1024, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02106 | -0.00114 | 0.01631 | +# | 44 | module.layer4.0.conv2.weight | (256, 512, 3, 3) | 1179648 | 1179648 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01159 | -0.00021 | 0.00906 | +# | 45 | module.layer4.0.conv3.weight | (2048, 256, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01445 | -0.00059 | 0.01123 | +# | 46 | module.layer4.0.downsample.0.weight | (2048, 1024, 1, 1) | 2097152 | 2097152 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00895 | -0.00014 | 0.00681 | +# | 47 | module.layer4.1.conv1.weight | (512, 2048, 1, 1) | 1048576 | 1048576 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01328 | -0.00062 | 0.01036 | +# | 48 | module.layer4.1.conv2.weight | (256, 512, 3, 3) | 1179648 | 1179648 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01152 | -0.00057 | 0.00906 | +# | 49 | module.layer4.1.conv3.weight | (2048, 256, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01414 | 0.00001 | 0.01094 | +# | 50 | module.layer4.2.conv1.weight | (512, 2048, 1, 1) | 1048576 | 1048576 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01636 | -0.00033 | 0.01284 | +# | 51 | module.layer4.2.conv2.weight | (256, 512, 3, 3) | 1179648 | 1179648 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01016 | -0.00044 | 0.00802 | +# | 52 | module.layer4.2.conv3.weight | (2048, 256, 1, 1) | 524288 | 524288 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.01318 | 0.00010 | 0.00993 | +# | 53 | module.fc.weight | (1000, 2048) | 2048000 | 2048000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03310 | 0.00000 | 0.02281 | +# | 54 | Total sparsity: | - | 17329344 | 17329344 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | +# +----+-------------------------------------+--------------------+---------------+----------------+------------+------------+----------+----------+----------+------------+---------+----------+------------+ +# 2018-12-04 23:32:08,902 - Total sparsity: 0.00 +# +# 2018-12-04 23:32:08,903 - --- validate (epoch=99)----------- +# 2018-12-04 23:32:08,903 - 50000 samples (256 per mini-batch) +# 2018-12-04 23:32:27,743 - Epoch: [99][ 50/ 195] Loss 0.683687 Top1 81.867188 Top5 95.937500 +# 2018-12-04 23:32:36,850 - Epoch: [99][ 100/ 195] Loss 0.810284 Top1 79.027344 Top5 94.648438 +# 2018-12-04 23:32:45,252 - Epoch: [99][ 150/ 195] Loss 0.934295 Top1 76.565104 Top5 93.072917 +# 2018-12-04 23:32:52,622 - ==> Top1: 75.654 Top5: 92.596 Loss: 0.978 +# +# 2018-12-04 23:32:52,693 - ==> Best Top1: 76.334 on Epoch: 0 +# 2018-12-04 23:32:52,694 - ==> Best Top1: 76.316 on Epoch: 1 +# 2018-12-04 23:32:52,694 - ==> Best Top1: 75.902 on Epoch: 3 +# 2018-12-04 23:32:52,694 - ==> Best Top1: 75.748 on Epoch: 94 <======== +# 2018-12-04 23:32:52,694 - ==> Best Top1: 75.732 on Epoch: 85 +# 2018-12-04 23:32:52,694 - ==> Best Top1: 75.728 on Epoch: 95 +# 2018-12-04 23:32:52,694 - ==> Best Top1: 75.698 on Epoch: 84 +# 2018-12-04 23:32:52,694 - ==> Best Top1: 75.674 on Epoch: 90 +# 2018-12-04 23:32:52,694 - ==> Best Top1: 75.664 on Epoch: 80 +# 2018-12-04 23:32:52,695 - ==> Best Top1: 75.654 on Epoch: 99 +# 2018-12-04 23:32:52,695 - Saving checkpoint to: logs/resnet50_filters___2018.12.02-224517/resnet50_filters_checkpoint.pth.tar +# 2018-12-04 23:32:53,013 - --- test --------------------- +# 2018-12-04 23:32:53,014 - 50000 samples (256 per mini-batch) +# 2018-12-04 23:33:12,090 - Test: [ 50/ 195] Loss 0.683687 Top1 81.867188 Top5 95.937500 +# 2018-12-04 23:33:20,491 - Test: [ 100/ 195] Loss 0.810284 Top1 79.027344 Top5 94.648438 +# 2018-12-04 23:33:28,604 - Test: [ 150/ 195] Loss 0.934295 Top1 76.565104 Top5 93.072917 +# 2018-12-04 23:33:36,294 - ==> Top1: 75.654 Top5: 92.596 Loss: 0.978 + +version: 1 + +pruners: + fc_pruner: + class: AutomatedGradualPruner + initial_sparsity : 0.05 + final_sparsity: 0.87 + weights: module.fc.weight + + filter_pruner: + class: L1RankedStructureParameterPruner_AGP + initial_sparsity : 0.05 + final_sparsity: 0.50 + group_type: Filters + weights: [module.layer1.0.conv2.weight, + module.layer1.1.conv2.weight, + module.layer1.2.conv2.weight, + module.layer2.0.conv2.weight, + module.layer2.1.conv2.weight, + module.layer2.2.conv2.weight, + module.layer2.3.conv2.weight, + module.layer3.0.conv2.weight, + module.layer3.1.conv2.weight, + module.layer3.2.conv2.weight, + module.layer3.3.conv2.weight, + module.layer3.4.conv2.weight, + module.layer3.5.conv2.weight, + module.layer4.0.conv2.weight, + module.layer4.1.conv2.weight, + module.layer4.2.conv2.weight] + + fine_pruner: + class: AutomatedGradualPruner + initial_sparsity : 0.05 + final_sparsity: 0.70 + weights: [ + module.layer4.0.conv2.weight, + module.layer4.0.conv3.weight, + module.layer4.0.downsample.0.weight, + module.layer4.1.conv1.weight, + module.layer4.1.conv2.weight, + module.layer4.1.conv3.weight, + module.layer4.2.conv1.weight, + module.layer4.2.conv2.weight, + module.layer4.2.conv3.weight] + +extensions: + net_thinner: + class: 'FilterRemover' + thinning_func_str: remove_filters + arch: 'resnet50' + dataset: 'imagenet' + +lr_schedulers: + pruning_lr: + class: ExponentialLR + gamma: 0.95 + +policies: + - pruner: + instance_name : filter_pruner + starting_epoch: 0 + ending_epoch: 30 + frequency: 2 + +# After completeing the pruning, we perform network thinning and continue fine-tuning. + - extension: + instance_name: net_thinner + epochs: [31] + + - lr_scheduler: + instance_name: pruning_lr + starting_epoch: 40 + ending_epoch: 100 + frequency: 1