From 1f289157e27c2127077eae5c6ae3fb1dd283c08d Mon Sep 17 00:00:00 2001 From: Aidan Sharpe <33038791+Adog64@users.noreply.github.com> Date: Wed, 24 Apr 2024 18:12:24 -0400 Subject: [PATCH] Converted old results to JSON format with metadata --- .../defense_filters.cpython-311.pyc | Bin 7064 -> 8607 bytes Filter_Analysis/defense_filters.py | 5 + Filter_Analysis/display_results.py | 2 +- Filter_Analysis/reformat_data.py | 30 + Filter_Analysis/results/mnist_fgsm.json | 691 +----------------- .../results/mnist_fgsm_reformatted.json | 672 +++++++++++++++++ Filter_Analysis/{fgsm.py => test_defenses.py} | 36 +- 7 files changed, 755 insertions(+), 681 deletions(-) create mode 100644 Filter_Analysis/reformat_data.py create mode 100644 Filter_Analysis/results/mnist_fgsm_reformatted.json rename Filter_Analysis/{fgsm.py => test_defenses.py} (89%) diff --git a/Filter_Analysis/__pycache__/defense_filters.cpython-311.pyc b/Filter_Analysis/__pycache__/defense_filters.cpython-311.pyc index 2c7862c60d7646e76b3444d1fe9751d7c91eda9a..e9b8315c59ce42bccdd87bc08fc22d2e82fab11b 100644 GIT binary patch delta 2717 zcmah~U2GIp6rMZ#JG(nO-Of%I7AcgLwo@Xs2|_7L>Dp2agjIeLg{aMzS?Ef;t#_u8 z);7xn4>iW5>li67vJXZ}B%!fNs=h$*#l(l%%_gjyH2NS9#FrXkV*H7oGyBu+0`Vqu z?wxzTbMAM~IrrR!{nz`wpYgnlf%c;+pZ`-#c{dT7YU@n;75f!()}i2 zr9p8+?@|H^54~I2qzKS^l+B6{dY)sHpd!M^tB_NSyyYR9j>x3oyg2!GbRt3eiKta)#+!*>%lWj;#dvnFhT3ioCymiNyk@|B|KU$m{T(h9PKd?Q}e`HS{ z>VZRC>Da^Eat&;=d=m=qXYcMKFuc1zys5w5c28j8=bn$?xZZ-}K-dP|ca4Br?zMbO zh)-X&_5{3OsNfooYUf6V)cACaZiWde=(}yrTkCMmcOg2ds_|%Is2qFR3iE;GgzX=4 zQbM!?an`X|G(iLS4_UxaI|>_$Ehu(?P;qX%*{EroN1b)M9k9GZPuP0(zZ`MWwj6Wr zV2Py9xw`5waP%1zTTwLX-@E$uZNuSq6ipyxj^bfxGl&8gPiV1(8rP<%fRimKcA}8= z!|tZy(L9hV%J|q&Vmz+KXg91r3!nO{e#g6C|I)p}x54iXmWrq+E>C&hCqT#H-Ic7R zgMcm&1`_&N zYE!U4+L_5%6L7gI)&=WuTvMw_B8QDvBLEo;s39db0kls8;39&)GJQ3<`En97(@cPK zRkf8tBW5A~2+G@4X2znBi%1==j>68c({L8WG7bNVb=u7^vsMjoA2Fw3gd;e_Yen_R6CP;!-s)N6PTH|X4SKLPJ4;e8r-soYG8Y!RI-Pnr(mD{#I% zU-C)`UYfjAubJo#pIGXcP!~CVx*>qFtjWo^pH9kCZHqBcNl!f zs@Re~z1XvKDI@mf#6H8*mrwHId@{?o8hmSMlcnyA7|Mwu!xLJwFy1y;p?FDUmexaO zz9HrDuB!P1IJT@~=;5lx!K(2~RY2AQKgHw_XqSF87<|Qi8_n`)mikJ5)aHblcxJL- zEN?I-NwY}*=?smHE0d$KUium=VP#Ms>9>PDf``tuap# Hrse+topxM4 delta 1405 zcmaJ>&1(}u6yM1v*=(B4N0XYi_AC9MT}A2#wpLnes_3PHR=ju#o6fdbnlEQIw9$h_ z{0Hh>Jhd0W56}vs(yN|4_y@?|JP8V-=&2z1-bT|Hiv#cXX5V`=^P73^?bd~dli|mj z7L;I2{75Z*3~z)xNp55G_)I(X-zPT}+D;YF0UD<&Xq6^t8|WbIpc?2ldWc3qhiH<< zKx>p-m-Nn!&Hgq8_LM=s?@1ePGl}^j$@bzo^)W25St4dgRpdM&5te)6UdLCg6v|ac zwV6@m$Su3dG&Bw(3?Yn) zs+Ou>Ed&oqONC6aWO0*CfEO9?&*HfQ$Qux1j3zXZ}#;#BMo zyxVr{0Xg2tOiMcS5Ic~$xE1eM`9E1@pR7Zu0gN@rx0*AVdEhGsnuw`(j+s1H%F||M zVkR?s zFw>h->v*%4&71k)ajo5;LQsa9Wgxf!{w0jr0azEe5AWQ?TGTxZ+u1G1KXbjwj+LOx z`!Q^$$$7D9j@Q-0^>PiCtBuaa9}>gANxj$HVs)!!cn*!&9&Vqwb?4@h;p@44pGpil zx=G&)OczSDk~hz>8(6eo;qLPbGx{R+ diff --git a/Filter_Analysis/defense_filters.py b/Filter_Analysis/defense_filters.py index 837ee8d..cba9d1a 100644 --- a/Filter_Analysis/defense_filters.py +++ b/Filter_Analysis/defense_filters.py @@ -9,6 +9,11 @@ def pttensor_to_images(data): images = None try: images = data.numpy().transpose(0,2,3,1) + except TypeError: + try: + images = data.cpu().numpy().transpose(0,2,3,1) + except RuntimeError: + images = data.cpu().detach().numpy().transpose(0,2,3,1) except RuntimeError: images = data.detach().numpy().transpose(0,2,3,1) diff --git a/Filter_Analysis/display_results.py b/Filter_Analysis/display_results.py index 9c3f301..f71eb5d 100644 --- a/Filter_Analysis/display_results.py +++ b/Filter_Analysis/display_results.py @@ -11,7 +11,7 @@ def main(): results = json.load(infile) filters = list(results.keys())[1:] - epsilons = np.arange( + #epsilons = np. for filter in filters: filter_performance = copy.deepcopy(results[filter]) for i in range(len(results["unfiltered"])): diff --git a/Filter_Analysis/reformat_data.py b/Filter_Analysis/reformat_data.py new file mode 100644 index 0000000..60415bd --- /dev/null +++ b/Filter_Analysis/reformat_data.py @@ -0,0 +1,30 @@ +import json +import numpy as np + +def reformat_data(): + results = {} + with open("results/mnist_fgsm.json", "r") as infile: + results = json.load(infile) + + reformatted_results = {} + + reformatted_results["dataset"] = "MNIST" + reformatted_results["attack"] = "FGSM" + + MAX_EPSILON = 0.3 + EPSILON_STEP = 0.025 + epsilons = np.arange(0.0, MAX_EPSILON+EPSILON_STEP, EPSILON_STEP) + + reformatted_results["epsilons"] = list(epsilons) + + reformatted_results["filters"] = {} + filters = list(results.keys())[1:] + for filter in filters: + reformatted_results["filters"][filter] = results[filter] + + reformatted_results_json = json.dumps(reformatted_results, indent=4) + with open("results/mnist_fgsm_reformatted.json", "w") as outfile: + outfile.write(reformatted_results_json) + +if __name__ == "__main__": + reformat_data() diff --git a/Filter_Analysis/results/mnist_fgsm.json b/Filter_Analysis/results/mnist_fgsm.json index 180c16e..0d79e48 100644 --- a/Filter_Analysis/results/mnist_fgsm.json +++ b/Filter_Analysis/results/mnist_fgsm.json @@ -1,668 +1,31 @@ { - "unfiltered": [ - 0.992, - 0.9796, - 0.96, - 0.926, - 0.8753, - 0.8104, - 0.7229, - 0.6207, - 0.5008, - 0.3894, - 0.2922, - 0.2149, - 0.1599 + "attack": "FGSM", + "dataset": "MNIST", + "epsilons": [ + 0.0, + 0.025, + 0.05, + 0.07500000000000001, + 0.1, + 0.125, + 0.15000000000000002, + 0.17500000000000002, + 0.2, + 0.225, + 0.25, + 0.275, + 0.30000000000000004 ], - "gaussian_blur": [ - [ - 0.992, - 0.9879, - 0.9682, - 0.7731, - 0.525 - ], - [ - 0.9796, - 0.9801, - 0.9512, - 0.7381, - 0.4862 - ], - [ - 0.96, - 0.9674, - 0.9271, - 0.6922, - 0.4446 - ], - [ - 0.926, - 0.946, - 0.8939, - 0.6427, - 0.3989 - ], - [ - 0.8753, - 0.9133, - 0.8516, - 0.5881, - 0.3603 - ], - [ - 0.8104, - 0.869, - 0.7989, - 0.5278, - 0.3263 - ], - [ - 0.7229, - 0.8135, - 0.7415, - 0.471, - 0.2968 - ], - [ - 0.6207, - 0.7456, - 0.6741, - 0.4224, - 0.2683 - ], - [ - 0.5008, - 0.6636, - 0.5983, - 0.3755, - 0.2453 - ], - [ - 0.3894, - 0.5821, - 0.5243, - 0.3359, - 0.2269 - ], - [ - 0.2922, - 0.505, - 0.4591, - 0.3034, - 0.2112 - ], - [ - 0.2149, - 0.429, - 0.3998, - 0.2743, - 0.1983 - ], - [ - 0.1599, - 0.3648, - 0.3481, - 0.2493, - 0.1884 + "filters": { + "gaussian_blur": [ + [ + 0.992, + 0.992, + 0.9879, + 0.9682, + 0.7731, + 0.525 + ] ] - ], - "gaussian_kuwahara": [ - [ - 0.9897, - 0.9766, - 0.9066, - 0.7355, - 0.5131 - ], - [ - 0.9808, - 0.9667, - 0.8909, - 0.7035, - 0.4824 - ], - [ - 0.9651, - 0.9547, - 0.87, - 0.6713, - 0.4538 - ], - [ - 0.9412, - 0.9334, - 0.8447, - 0.6354, - 0.426 - ], - [ - 0.9035, - 0.9107, - 0.8123, - 0.597, - 0.3915 - ], - [ - 0.8539, - 0.8785, - 0.7751, - 0.5616, - 0.362 - ], - [ - 0.7925, - 0.8328, - 0.7328, - 0.5236, - 0.3344 - ], - [ - 0.7078, - 0.7808, - 0.6816, - 0.4868, - 0.309 - ], - [ - 0.6125, - 0.7179, - 0.6301, - 0.4513, - 0.2865 - ], - [ - 0.4979, - 0.646, - 0.5773, - 0.4242, - 0.2702 - ], - [ - 0.3927, - 0.564, - 0.5197, - 0.3859, - 0.2493 - ], - [ - 0.3023, - 0.4761, - 0.4594, - 0.3494, - 0.2354 - ], - [ - 0.2289, - 0.3839, - 0.3981, - 0.3182, - 0.2232 - ] - ], - "mean_kuwahara": [ - [ - 0.988, - 0.7536, - 0.3667, - 0.1763, - 0.1339 - ], - [ - 0.9795, - 0.7359, - 0.3496, - 0.171, - 0.1318 - ], - [ - 0.965, - 0.7129, - 0.3295, - 0.1637, - 0.1286 - ], - [ - 0.946, - 0.6871, - 0.3119, - 0.1578, - 0.1244 - ], - [ - 0.916, - 0.6617, - 0.2841, - 0.1497, - 0.1228 - ], - [ - 0.8746, - 0.6317, - 0.2587, - 0.1422, - 0.1211 - ], - [ - 0.8235, - 0.6019, - 0.2395, - 0.136, - 0.1193 - ], - [ - 0.7499, - 0.5699, - 0.2253, - 0.134, - 0.1164 - ], - [ - 0.665, - 0.542, - 0.2168, - 0.1335, - 0.1138 - ], - [ - 0.5642, - 0.5087, - 0.2064, - 0.1328, - 0.1129 - ], - [ - 0.4739, - 0.4773, - 0.1993, - 0.1306, - 0.1145 - ], - [ - 0.3638, - 0.437, - 0.1921, - 0.1309, - 0.1159 - ], - [ - 0.2659, - 0.3912, - 0.1854, - 0.1307, - 0.1166 - ] - ], - "random_noise": [ - [ - 0.9913, - 0.9899, - 0.9872, - 0.9719, - 0.9226 - ], - [ - 0.9793, - 0.9782, - 0.9711, - 0.9453, - 0.8802 - ], - [ - 0.9603, - 0.9568, - 0.9436, - 0.9049, - 0.8184 - ], - [ - 0.9253, - 0.9183, - 0.895, - 0.8392, - 0.7432 - ], - [ - 0.8743, - 0.8653, - 0.8309, - 0.7656, - 0.6606 - ], - [ - 0.809, - 0.7948, - 0.7486, - 0.6709, - 0.5588 - ], - [ - 0.721, - 0.6999, - 0.6485, - 0.5625, - 0.4577 - ], - [ - 0.6157, - 0.5881, - 0.5377, - 0.4548, - 0.3647 - ], - [ - 0.5005, - 0.4802, - 0.4267, - 0.3571, - 0.2885 - ], - [ - 0.385, - 0.3668, - 0.3295, - 0.2696, - 0.2223 - ], - [ - 0.2918, - 0.2758, - 0.244, - 0.2039, - 0.1724 - ], - [ - 0.215, - 0.2016, - 0.1832, - 0.1555, - 0.1326 - ], - [ - 0.1591, - 0.154, - 0.1371, - 0.1163, - 0.1021 - ] - ], - "bilateral_filter": [ - [ - 0.9887, - 0.9887, - 0.9391, - 0.5584, - 0.2568 - ], - [ - 0.9809, - 0.9809, - 0.9184, - 0.5198, - 0.241 - ], - [ - 0.9695, - 0.9695, - 0.8902, - 0.4831, - 0.2245 - ], - [ - 0.9482, - 0.9482, - 0.8533, - 0.4436, - 0.2079 - ], - [ - 0.9142, - 0.9142, - 0.8133, - 0.4019, - 0.1915 - ], - [ - 0.8714, - 0.8714, - 0.7656, - 0.3641, - 0.1792 - ], - [ - 0.8169, - 0.8169, - 0.7098, - 0.3299, - 0.1681 - ], - [ - 0.7477, - 0.7477, - 0.641, - 0.2978, - 0.161 - ], - [ - 0.6619, - 0.6619, - 0.5683, - 0.2723, - 0.1563 - ], - [ - 0.5767, - 0.5767, - 0.5003, - 0.2476, - 0.1517 - ], - [ - 0.4922, - 0.4922, - 0.4381, - 0.2288, - 0.1484 - ], - [ - 0.4133, - 0.4133, - 0.3836, - 0.2126, - 0.146 - ], - [ - 0.3468, - 0.3468, - 0.3364, - 0.1999, - 0.1444 - ] - ], - "bit_depth": [ - [ - 0.9894, - 0.9913, - 0.9916, - 0.992, - 0.992 - ], - [ - 0.9862, - 0.9823, - 0.9807, - 0.9796, - 0.9796 - ], - [ - 0.9808, - 0.9781, - 0.965, - 0.9604, - 0.96 - ], - [ - 0.9744, - 0.9228, - 0.9219, - 0.926, - 0.926 - ], - [ - 0.9424, - 0.8818, - 0.8747, - 0.8751, - 0.8753 - ], - [ - 0.9307, - 0.8621, - 0.821, - 0.8094, - 0.8104 - ], - [ - 0.9157, - 0.8408, - 0.7427, - 0.7235, - 0.7229 - ], - [ - 0.8972, - 0.7794, - 0.6554, - 0.6229, - 0.6207 - ], - [ - 0.8799, - 0.7496, - 0.559, - 0.5046, - 0.5008 - ], - [ - 0.8581, - 0.5289, - 0.4547, - 0.3914, - 0.3894 - ], - [ - 0.7603, - 0.4301, - 0.3113, - 0.2927, - 0.2922 - ], - [ - 0.7227, - 0.3992, - 0.2414, - 0.2168, - 0.2149 - ], - [ - 0.2624, - 0.2091, - 0.1874, - 0.161, - 0.1599 - ] - ], - "threshold_filter": [ - [ - 0.982, - 0.9817, - 0.9799, - 0.9713, - 0.9502 - ], - [ - 0.978, - 0.9755, - 0.9751, - 0.9655, - 0.9334 - ], - [ - 0.9728, - 0.9713, - 0.9696, - 0.9578, - 0.9077 - ], - [ - 0.9678, - 0.9668, - 0.9645, - 0.9508, - 0.1837 - ], - [ - 0.9644, - 0.9604, - 0.9583, - 0.9407, - 0.1818 - ], - [ - 0.9586, - 0.9537, - 0.9477, - 0.9238, - 0.1817 - ], - [ - 0.9522, - 0.9458, - 0.9343, - 0.9032, - 0.1845 - ], - [ - 0.9418, - 0.9387, - 0.9236, - 0.8766, - 0.1849 - ], - [ - 0.9331, - 0.9297, - 0.9108, - 0.8358, - 0.1869 - ], - [ - 0.9215, - 0.9188, - 0.8927, - 0.2164, - 0.1904 - ], - [ - 0.9079, - 0.9053, - 0.8758, - 0.223, - 0.1927 - ], - [ - 0.8943, - 0.8882, - 0.8508, - 0.2275, - 0.1979 - ], - [ - 0.8761, - 0.8687, - 0.8142, - 0.2348, - 0.2025 - ] - ] + } } \ No newline at end of file diff --git a/Filter_Analysis/results/mnist_fgsm_reformatted.json b/Filter_Analysis/results/mnist_fgsm_reformatted.json new file mode 100644 index 0000000..f7bf76a --- /dev/null +++ b/Filter_Analysis/results/mnist_fgsm_reformatted.json @@ -0,0 +1,672 @@ +{ + "dataset": "MNIST", + "attack": "FGSM", + "epsilons": [ + 0.0, + 0.025, + 0.05, + 0.07500000000000001, + 0.1, + 0.125, + 0.15000000000000002, + 0.17500000000000002, + 0.2, + 0.225, + 0.25, + 0.275, + 0.30000000000000004 + ], + "filters": { + "gaussian_blur": [ + [ + 0.992, + 0.9879, + 0.9682, + 0.7731, + 0.525 + ], + [ + 0.9796, + 0.9801, + 0.9512, + 0.7381, + 0.4862 + ], + [ + 0.96, + 0.9674, + 0.9271, + 0.6922, + 0.4446 + ], + [ + 0.926, + 0.946, + 0.8939, + 0.6427, + 0.3989 + ], + [ + 0.8753, + 0.9133, + 0.8516, + 0.5881, + 0.3603 + ], + [ + 0.8104, + 0.869, + 0.7989, + 0.5278, + 0.3263 + ], + [ + 0.7229, + 0.8135, + 0.7415, + 0.471, + 0.2968 + ], + [ + 0.6207, + 0.7456, + 0.6741, + 0.4224, + 0.2683 + ], + [ + 0.5008, + 0.6636, + 0.5983, + 0.3755, + 0.2453 + ], + [ + 0.3894, + 0.5821, + 0.5243, + 0.3359, + 0.2269 + ], + [ + 0.2922, + 0.505, + 0.4591, + 0.3034, + 0.2112 + ], + [ + 0.2149, + 0.429, + 0.3998, + 0.2743, + 0.1983 + ], + [ + 0.1599, + 0.3648, + 0.3481, + 0.2493, + 0.1884 + ] + ], + "gaussian_kuwahara": [ + [ + 0.9897, + 0.9766, + 0.9066, + 0.7355, + 0.5131 + ], + [ + 0.9808, + 0.9667, + 0.8909, + 0.7035, + 0.4824 + ], + [ + 0.9651, + 0.9547, + 0.87, + 0.6713, + 0.4538 + ], + [ + 0.9412, + 0.9334, + 0.8447, + 0.6354, + 0.426 + ], + [ + 0.9035, + 0.9107, + 0.8123, + 0.597, + 0.3915 + ], + [ + 0.8539, + 0.8785, + 0.7751, + 0.5616, + 0.362 + ], + [ + 0.7925, + 0.8328, + 0.7328, + 0.5236, + 0.3344 + ], + [ + 0.7078, + 0.7808, + 0.6816, + 0.4868, + 0.309 + ], + [ + 0.6125, + 0.7179, + 0.6301, + 0.4513, + 0.2865 + ], + [ + 0.4979, + 0.646, + 0.5773, + 0.4242, + 0.2702 + ], + [ + 0.3927, + 0.564, + 0.5197, + 0.3859, + 0.2493 + ], + [ + 0.3023, + 0.4761, + 0.4594, + 0.3494, + 0.2354 + ], + [ + 0.2289, + 0.3839, + 0.3981, + 0.3182, + 0.2232 + ] + ], + "mean_kuwahara": [ + [ + 0.988, + 0.7536, + 0.3667, + 0.1763, + 0.1339 + ], + [ + 0.9795, + 0.7359, + 0.3496, + 0.171, + 0.1318 + ], + [ + 0.965, + 0.7129, + 0.3295, + 0.1637, + 0.1286 + ], + [ + 0.946, + 0.6871, + 0.3119, + 0.1578, + 0.1244 + ], + [ + 0.916, + 0.6617, + 0.2841, + 0.1497, + 0.1228 + ], + [ + 0.8746, + 0.6317, + 0.2587, + 0.1422, + 0.1211 + ], + [ + 0.8235, + 0.6019, + 0.2395, + 0.136, + 0.1193 + ], + [ + 0.7499, + 0.5699, + 0.2253, + 0.134, + 0.1164 + ], + [ + 0.665, + 0.542, + 0.2168, + 0.1335, + 0.1138 + ], + [ + 0.5642, + 0.5087, + 0.2064, + 0.1328, + 0.1129 + ], + [ + 0.4739, + 0.4773, + 0.1993, + 0.1306, + 0.1145 + ], + [ + 0.3638, + 0.437, + 0.1921, + 0.1309, + 0.1159 + ], + [ + 0.2659, + 0.3912, + 0.1854, + 0.1307, + 0.1166 + ] + ], + "random_noise": [ + [ + 0.9913, + 0.9899, + 0.9872, + 0.9719, + 0.9226 + ], + [ + 0.9793, + 0.9782, + 0.9711, + 0.9453, + 0.8802 + ], + [ + 0.9603, + 0.9568, + 0.9436, + 0.9049, + 0.8184 + ], + [ + 0.9253, + 0.9183, + 0.895, + 0.8392, + 0.7432 + ], + [ + 0.8743, + 0.8653, + 0.8309, + 0.7656, + 0.6606 + ], + [ + 0.809, + 0.7948, + 0.7486, + 0.6709, + 0.5588 + ], + [ + 0.721, + 0.6999, + 0.6485, + 0.5625, + 0.4577 + ], + [ + 0.6157, + 0.5881, + 0.5377, + 0.4548, + 0.3647 + ], + [ + 0.5005, + 0.4802, + 0.4267, + 0.3571, + 0.2885 + ], + [ + 0.385, + 0.3668, + 0.3295, + 0.2696, + 0.2223 + ], + [ + 0.2918, + 0.2758, + 0.244, + 0.2039, + 0.1724 + ], + [ + 0.215, + 0.2016, + 0.1832, + 0.1555, + 0.1326 + ], + [ + 0.1591, + 0.154, + 0.1371, + 0.1163, + 0.1021 + ] + ], + "bilateral_filter": [ + [ + 0.9887, + 0.9887, + 0.9391, + 0.5584, + 0.2568 + ], + [ + 0.9809, + 0.9809, + 0.9184, + 0.5198, + 0.241 + ], + [ + 0.9695, + 0.9695, + 0.8902, + 0.4831, + 0.2245 + ], + [ + 0.9482, + 0.9482, + 0.8533, + 0.4436, + 0.2079 + ], + [ + 0.9142, + 0.9142, + 0.8133, + 0.4019, + 0.1915 + ], + [ + 0.8714, + 0.8714, + 0.7656, + 0.3641, + 0.1792 + ], + [ + 0.8169, + 0.8169, + 0.7098, + 0.3299, + 0.1681 + ], + [ + 0.7477, + 0.7477, + 0.641, + 0.2978, + 0.161 + ], + [ + 0.6619, + 0.6619, + 0.5683, + 0.2723, + 0.1563 + ], + [ + 0.5767, + 0.5767, + 0.5003, + 0.2476, + 0.1517 + ], + [ + 0.4922, + 0.4922, + 0.4381, + 0.2288, + 0.1484 + ], + [ + 0.4133, + 0.4133, + 0.3836, + 0.2126, + 0.146 + ], + [ + 0.3468, + 0.3468, + 0.3364, + 0.1999, + 0.1444 + ] + ], + "bit_depth": [ + [ + 0.9894, + 0.9913, + 0.9916, + 0.992, + 0.992 + ], + [ + 0.9862, + 0.9823, + 0.9807, + 0.9796, + 0.9796 + ], + [ + 0.9808, + 0.9781, + 0.965, + 0.9604, + 0.96 + ], + [ + 0.9744, + 0.9228, + 0.9219, + 0.926, + 0.926 + ], + [ + 0.9424, + 0.8818, + 0.8747, + 0.8751, + 0.8753 + ], + [ + 0.9307, + 0.8621, + 0.821, + 0.8094, + 0.8104 + ], + [ + 0.9157, + 0.8408, + 0.7427, + 0.7235, + 0.7229 + ], + [ + 0.8972, + 0.7794, + 0.6554, + 0.6229, + 0.6207 + ], + [ + 0.8799, + 0.7496, + 0.559, + 0.5046, + 0.5008 + ], + [ + 0.8581, + 0.5289, + 0.4547, + 0.3914, + 0.3894 + ], + [ + 0.7603, + 0.4301, + 0.3113, + 0.2927, + 0.2922 + ], + [ + 0.7227, + 0.3992, + 0.2414, + 0.2168, + 0.2149 + ], + [ + 0.2624, + 0.2091, + 0.1874, + 0.161, + 0.1599 + ] + ], + "threshold_filter": [ + [ + 0.982, + 0.9817, + 0.9799, + 0.9713, + 0.9502 + ], + [ + 0.978, + 0.9755, + 0.9751, + 0.9655, + 0.9334 + ], + [ + 0.9728, + 0.9713, + 0.9696, + 0.9578, + 0.9077 + ], + [ + 0.9678, + 0.9668, + 0.9645, + 0.9508, + 0.1837 + ], + [ + 0.9644, + 0.9604, + 0.9583, + 0.9407, + 0.1818 + ], + [ + 0.9586, + 0.9537, + 0.9477, + 0.9238, + 0.1817 + ], + [ + 0.9522, + 0.9458, + 0.9343, + 0.9032, + 0.1845 + ], + [ + 0.9418, + 0.9387, + 0.9236, + 0.8766, + 0.1849 + ], + [ + 0.9331, + 0.9297, + 0.9108, + 0.8358, + 0.1869 + ], + [ + 0.9215, + 0.9188, + 0.8927, + 0.2164, + 0.1904 + ], + [ + 0.9079, + 0.9053, + 0.8758, + 0.223, + 0.1927 + ], + [ + 0.8943, + 0.8882, + 0.8508, + 0.2275, + 0.1979 + ], + [ + 0.8761, + 0.8687, + 0.8142, + 0.2348, + 0.2025 + ] + ] + } +} \ No newline at end of file diff --git a/Filter_Analysis/fgsm.py b/Filter_Analysis/test_defenses.py similarity index 89% rename from Filter_Analysis/fgsm.py rename to Filter_Analysis/test_defenses.py index 9aceb5a..9f3d09b 100644 --- a/Filter_Analysis/fgsm.py +++ b/Filter_Analysis/test_defenses.py @@ -15,10 +15,13 @@ import defense_filters -TESTED_STRENGTH_COUNT = 5 +ATTACK = "FGSM" +DATASET = "MNIST" MAX_EPSILON = 0.3 EPSILON_STEP = 0.025 + +TESTED_STRENGTH_COUNT = 5 epsilons = np.arange(0.0, MAX_EPSILON+EPSILON_STEP, EPSILON_STEP) pretrained_model = "mnist_cnn_unfiltered.pt" use_cuda=False @@ -87,7 +90,7 @@ def test(model, device, test_loader, epsilon, filter): test_step = 0 for data, target in test_loader: sys.stdout.write("\033[K") - print(filter, f"Epsilon: {epsilon}", "[" + "="*int(1 + 20*test_step/len(test_loader)) + " "*(20 - int(20*test_step/len(test_loader))) + "]", f"{100*test_step/len(test_loader)}%", end='\r') + print(filter, f"Epsilon: {epsilon}", "[" + "="*int(1 + 20*test_step/len(test_loader)) + " "*(20 - int(20*test_step/len(test_loader))) + "]", f"{100*test_step/len(test_loader):.3f}%", end='\r') test_step += 1 data, target = data.to(device), target.to(device) @@ -161,29 +164,30 @@ def test(model, device, test_loader, epsilon, filter): #for i in range(TESTED_STRENGTH_COUNT): # strength = i+1 # print(f"{filter}({strength}) = {filtered_correct_counts[i]} / {len(test_loader)} = {filtered_accuracies[i]}") - - return unfiltered_acc, filtered_accuracies + accuracies = [unfiltered_acc] + accuracies.extend(filtered_accuracies) + return accuracies -accuracies = {} +results = {} +results["attack"] = ATTACK +results["dataset"] = DATASET +results["epsilons"] = list(epsilons) +results["filters"] = {} + filters = ("gaussian_blur", "gaussian_kuwahara", "mean_kuwahara", "random_noise", "bilateral_filter", "bit_depth", "threshold_filter") for filter in filters: for eps in epsilons: - unfiltered_accuracy, filtered_accuracy = test(model, device, test_loader, eps, filter) + accuracies = test(model, device, test_loader, eps, filter) - if list(accuracies.keys()).count("unfiltered") == 0: - accuracies["unfiltered"] = [] - if len(accuracies["unfiltered"]) < len(epsilons): - accuracies["unfiltered"].append(unfiltered_accuracy) + if list(results["filters"].keys()).count(filter) == 0: + results["filters"][filter] = [] + results["filters"][filter].append(accuracies) - if list(accuracies.keys()).count(filter) == 0: - accuracies[filter] = [] - accuracies[filter].append(filtered_accuracy) - - accuracies_json = json.dumps(accuracies, indent=4) + results_json = json.dumps(results, indent=4) with open("results/mnist_fgsm.json", "w") as outfile: - outfile.write(accuracies_json) + outfile.write(results_json) # Plot the results #plt.figure(figsize=(16,9))