From 610d8b867864e54975875e678b516dfe7515ffa9 Mon Sep 17 00:00:00 2001 From: Adog64 Date: Wed, 24 Apr 2024 12:16:09 -0400 Subject: [PATCH] got data for fgsm mnist, working on displaying it --- Filter_Analysis/display_results.py | 36 + Filter_Analysis/fgsm.py | 6 +- .../BilateralPerformance50Sigma.png | Bin .../BitDepthReducePerformance.png | Bin .../GaussianBlurPerformance.png | Bin .../GaussianKuwaharaPerformance.png | Bin .../MeanKuwaharaPerformance.png | Bin .../RandomNoisePerformance.png | Bin .../ThresholdFilterPerformance.png | Bin Filter_Analysis/results/mnist_fgsm.json | 668 ++++++++++++++++++ 10 files changed, 709 insertions(+), 1 deletion(-) create mode 100644 Filter_Analysis/display_results.py rename Filter_Analysis/{Images => images}/BilateralPerformance50Sigma.png (100%) rename Filter_Analysis/{Images => images}/BitDepthReducePerformance.png (100%) rename Filter_Analysis/{Images => images}/GaussianBlurPerformance.png (100%) rename Filter_Analysis/{Images => images}/GaussianKuwaharaPerformance.png (100%) rename Filter_Analysis/{Images => images}/MeanKuwaharaPerformance.png (100%) rename Filter_Analysis/{Images => images}/RandomNoisePerformance.png (100%) rename Filter_Analysis/{Images => images}/ThresholdFilterPerformance.png (100%) create mode 100644 Filter_Analysis/results/mnist_fgsm.json diff --git a/Filter_Analysis/display_results.py b/Filter_Analysis/display_results.py new file mode 100644 index 0000000..9c3f301 --- /dev/null +++ b/Filter_Analysis/display_results.py @@ -0,0 +1,36 @@ +import json +import numpy as np +import matplotlib.pyplot as plt +import copy + + + +def main(): + results = {} + with open("results/mnist_fgsm.json", "r") as infile: + results = json.load(infile) + + filters = list(results.keys())[1:] + epsilons = np.arange( + for filter in filters: + filter_performance = copy.deepcopy(results[filter]) + for i in range(len(results["unfiltered"])): + filter_performance[i].insert(0, results["unfiltered"][i]) + + plt.figure(figsize=(16,9)) + plt.plot(epsilons, unfiltered_accuracies, label="Attacked Accuracy") + + for i in range(TESTED_STRENGTH_COUNT): + filtered_accuracy = [filter_eps[i] for filter_eps in filter_performance] + plt.plot(epsilons, filtered_accuracy, label=f"Strength = {i}") + + plt.legend(loc="upper right") + plt.title(f"{filter} Performance") + plt.xlabel("Attack Strength ($\\epsilon$)") + plt.ylabel("Accuracy") + plt.show() + + + +if __name__ == "__main__": + main() diff --git a/Filter_Analysis/fgsm.py b/Filter_Analysis/fgsm.py index 2a16eb3..9aceb5a 100644 --- a/Filter_Analysis/fgsm.py +++ b/Filter_Analysis/fgsm.py @@ -9,6 +9,7 @@ import matplotlib.pyplot as plt from mnist import Net import json +import sys import defense_filters @@ -85,6 +86,7 @@ def test(model, device, test_loader, epsilon, filter): test_step = 0 for data, target in test_loader: + sys.stdout.write("\033[K") print(filter, f"Epsilon: {epsilon}", "[" + "="*int(1 + 20*test_step/len(test_loader)) + " "*(20 - int(20*test_step/len(test_loader))) + "]", f"{100*test_step/len(test_loader)}%", end='\r') test_step += 1 @@ -179,7 +181,9 @@ for filter in filters: accuracies[filter] = [] accuracies[filter].append(filtered_accuracy) -json.dump(accuracies, "fgsm_mnist_accuracies.json") + accuracies_json = json.dumps(accuracies, indent=4) + with open("results/mnist_fgsm.json", "w") as outfile: + outfile.write(accuracies_json) # Plot the results #plt.figure(figsize=(16,9)) diff --git a/Filter_Analysis/Images/BilateralPerformance50Sigma.png b/Filter_Analysis/images/BilateralPerformance50Sigma.png similarity index 100% rename from Filter_Analysis/Images/BilateralPerformance50Sigma.png rename to Filter_Analysis/images/BilateralPerformance50Sigma.png diff --git a/Filter_Analysis/Images/BitDepthReducePerformance.png b/Filter_Analysis/images/BitDepthReducePerformance.png similarity index 100% rename from Filter_Analysis/Images/BitDepthReducePerformance.png rename to Filter_Analysis/images/BitDepthReducePerformance.png diff --git a/Filter_Analysis/Images/GaussianBlurPerformance.png b/Filter_Analysis/images/GaussianBlurPerformance.png similarity index 100% rename from Filter_Analysis/Images/GaussianBlurPerformance.png rename to Filter_Analysis/images/GaussianBlurPerformance.png diff --git a/Filter_Analysis/Images/GaussianKuwaharaPerformance.png b/Filter_Analysis/images/GaussianKuwaharaPerformance.png similarity index 100% rename from Filter_Analysis/Images/GaussianKuwaharaPerformance.png rename to Filter_Analysis/images/GaussianKuwaharaPerformance.png diff --git a/Filter_Analysis/Images/MeanKuwaharaPerformance.png b/Filter_Analysis/images/MeanKuwaharaPerformance.png similarity index 100% rename from Filter_Analysis/Images/MeanKuwaharaPerformance.png rename to Filter_Analysis/images/MeanKuwaharaPerformance.png diff --git a/Filter_Analysis/Images/RandomNoisePerformance.png b/Filter_Analysis/images/RandomNoisePerformance.png similarity index 100% rename from Filter_Analysis/Images/RandomNoisePerformance.png rename to Filter_Analysis/images/RandomNoisePerformance.png diff --git a/Filter_Analysis/Images/ThresholdFilterPerformance.png b/Filter_Analysis/images/ThresholdFilterPerformance.png similarity index 100% rename from Filter_Analysis/Images/ThresholdFilterPerformance.png rename to Filter_Analysis/images/ThresholdFilterPerformance.png diff --git a/Filter_Analysis/results/mnist_fgsm.json b/Filter_Analysis/results/mnist_fgsm.json new file mode 100644 index 0000000..180c16e --- /dev/null +++ b/Filter_Analysis/results/mnist_fgsm.json @@ -0,0 +1,668 @@ +{ + "unfiltered": [ + 0.992, + 0.9796, + 0.96, + 0.926, + 0.8753, + 0.8104, + 0.7229, + 0.6207, + 0.5008, + 0.3894, + 0.2922, + 0.2149, + 0.1599 + ], + "gaussian_blur": [ + [ + 0.992, + 0.9879, + 0.9682, + 0.7731, + 0.525 + ], + [ + 0.9796, + 0.9801, + 0.9512, + 0.7381, + 0.4862 + ], + [ + 0.96, + 0.9674, + 0.9271, + 0.6922, + 0.4446 + ], + [ + 0.926, + 0.946, + 0.8939, + 0.6427, + 0.3989 + ], + [ + 0.8753, + 0.9133, + 0.8516, + 0.5881, + 0.3603 + ], + [ + 0.8104, + 0.869, + 0.7989, + 0.5278, + 0.3263 + ], + [ + 0.7229, + 0.8135, + 0.7415, + 0.471, + 0.2968 + ], + [ + 0.6207, + 0.7456, + 0.6741, + 0.4224, + 0.2683 + ], + [ + 0.5008, + 0.6636, + 0.5983, + 0.3755, + 0.2453 + ], + [ + 0.3894, + 0.5821, + 0.5243, + 0.3359, + 0.2269 + ], + [ + 0.2922, + 0.505, + 0.4591, + 0.3034, + 0.2112 + ], + [ + 0.2149, + 0.429, + 0.3998, + 0.2743, + 0.1983 + ], + [ + 0.1599, + 0.3648, + 0.3481, + 0.2493, + 0.1884 + ] + ], + "gaussian_kuwahara": [ + [ + 0.9897, + 0.9766, + 0.9066, + 0.7355, + 0.5131 + ], + [ + 0.9808, + 0.9667, + 0.8909, + 0.7035, + 0.4824 + ], + [ + 0.9651, + 0.9547, + 0.87, + 0.6713, + 0.4538 + ], + [ + 0.9412, + 0.9334, + 0.8447, + 0.6354, + 0.426 + ], + [ + 0.9035, + 0.9107, + 0.8123, + 0.597, + 0.3915 + ], + [ + 0.8539, + 0.8785, + 0.7751, + 0.5616, + 0.362 + ], + [ + 0.7925, + 0.8328, + 0.7328, + 0.5236, + 0.3344 + ], + [ + 0.7078, + 0.7808, + 0.6816, + 0.4868, + 0.309 + ], + [ + 0.6125, + 0.7179, + 0.6301, + 0.4513, + 0.2865 + ], + [ + 0.4979, + 0.646, + 0.5773, + 0.4242, + 0.2702 + ], + [ + 0.3927, + 0.564, + 0.5197, + 0.3859, + 0.2493 + ], + [ + 0.3023, + 0.4761, + 0.4594, + 0.3494, + 0.2354 + ], + [ + 0.2289, + 0.3839, + 0.3981, + 0.3182, + 0.2232 + ] + ], + "mean_kuwahara": [ + [ + 0.988, + 0.7536, + 0.3667, + 0.1763, + 0.1339 + ], + [ + 0.9795, + 0.7359, + 0.3496, + 0.171, + 0.1318 + ], + [ + 0.965, + 0.7129, + 0.3295, + 0.1637, + 0.1286 + ], + [ + 0.946, + 0.6871, + 0.3119, + 0.1578, + 0.1244 + ], + [ + 0.916, + 0.6617, + 0.2841, + 0.1497, + 0.1228 + ], + [ + 0.8746, + 0.6317, + 0.2587, + 0.1422, + 0.1211 + ], + [ + 0.8235, + 0.6019, + 0.2395, + 0.136, + 0.1193 + ], + [ + 0.7499, + 0.5699, + 0.2253, + 0.134, + 0.1164 + ], + [ + 0.665, + 0.542, + 0.2168, + 0.1335, + 0.1138 + ], + [ + 0.5642, + 0.5087, + 0.2064, + 0.1328, + 0.1129 + ], + [ + 0.4739, + 0.4773, + 0.1993, + 0.1306, + 0.1145 + ], + [ + 0.3638, + 0.437, + 0.1921, + 0.1309, + 0.1159 + ], + [ + 0.2659, + 0.3912, + 0.1854, + 0.1307, + 0.1166 + ] + ], + "random_noise": [ + [ + 0.9913, + 0.9899, + 0.9872, + 0.9719, + 0.9226 + ], + [ + 0.9793, + 0.9782, + 0.9711, + 0.9453, + 0.8802 + ], + [ + 0.9603, + 0.9568, + 0.9436, + 0.9049, + 0.8184 + ], + [ + 0.9253, + 0.9183, + 0.895, + 0.8392, + 0.7432 + ], + [ + 0.8743, + 0.8653, + 0.8309, + 0.7656, + 0.6606 + ], + [ + 0.809, + 0.7948, + 0.7486, + 0.6709, + 0.5588 + ], + [ + 0.721, + 0.6999, + 0.6485, + 0.5625, + 0.4577 + ], + [ + 0.6157, + 0.5881, + 0.5377, + 0.4548, + 0.3647 + ], + [ + 0.5005, + 0.4802, + 0.4267, + 0.3571, + 0.2885 + ], + [ + 0.385, + 0.3668, + 0.3295, + 0.2696, + 0.2223 + ], + [ + 0.2918, + 0.2758, + 0.244, + 0.2039, + 0.1724 + ], + [ + 0.215, + 0.2016, + 0.1832, + 0.1555, + 0.1326 + ], + [ + 0.1591, + 0.154, + 0.1371, + 0.1163, + 0.1021 + ] + ], + "bilateral_filter": [ + [ + 0.9887, + 0.9887, + 0.9391, + 0.5584, + 0.2568 + ], + [ + 0.9809, + 0.9809, + 0.9184, + 0.5198, + 0.241 + ], + [ + 0.9695, + 0.9695, + 0.8902, + 0.4831, + 0.2245 + ], + [ + 0.9482, + 0.9482, + 0.8533, + 0.4436, + 0.2079 + ], + [ + 0.9142, + 0.9142, + 0.8133, + 0.4019, + 0.1915 + ], + [ + 0.8714, + 0.8714, + 0.7656, + 0.3641, + 0.1792 + ], + [ + 0.8169, + 0.8169, + 0.7098, + 0.3299, + 0.1681 + ], + [ + 0.7477, + 0.7477, + 0.641, + 0.2978, + 0.161 + ], + [ + 0.6619, + 0.6619, + 0.5683, + 0.2723, + 0.1563 + ], + [ + 0.5767, + 0.5767, + 0.5003, + 0.2476, + 0.1517 + ], + [ + 0.4922, + 0.4922, + 0.4381, + 0.2288, + 0.1484 + ], + [ + 0.4133, + 0.4133, + 0.3836, + 0.2126, + 0.146 + ], + [ + 0.3468, + 0.3468, + 0.3364, + 0.1999, + 0.1444 + ] + ], + "bit_depth": [ + [ + 0.9894, + 0.9913, + 0.9916, + 0.992, + 0.992 + ], + [ + 0.9862, + 0.9823, + 0.9807, + 0.9796, + 0.9796 + ], + [ + 0.9808, + 0.9781, + 0.965, + 0.9604, + 0.96 + ], + [ + 0.9744, + 0.9228, + 0.9219, + 0.926, + 0.926 + ], + [ + 0.9424, + 0.8818, + 0.8747, + 0.8751, + 0.8753 + ], + [ + 0.9307, + 0.8621, + 0.821, + 0.8094, + 0.8104 + ], + [ + 0.9157, + 0.8408, + 0.7427, + 0.7235, + 0.7229 + ], + [ + 0.8972, + 0.7794, + 0.6554, + 0.6229, + 0.6207 + ], + [ + 0.8799, + 0.7496, + 0.559, + 0.5046, + 0.5008 + ], + [ + 0.8581, + 0.5289, + 0.4547, + 0.3914, + 0.3894 + ], + [ + 0.7603, + 0.4301, + 0.3113, + 0.2927, + 0.2922 + ], + [ + 0.7227, + 0.3992, + 0.2414, + 0.2168, + 0.2149 + ], + [ + 0.2624, + 0.2091, + 0.1874, + 0.161, + 0.1599 + ] + ], + "threshold_filter": [ + [ + 0.982, + 0.9817, + 0.9799, + 0.9713, + 0.9502 + ], + [ + 0.978, + 0.9755, + 0.9751, + 0.9655, + 0.9334 + ], + [ + 0.9728, + 0.9713, + 0.9696, + 0.9578, + 0.9077 + ], + [ + 0.9678, + 0.9668, + 0.9645, + 0.9508, + 0.1837 + ], + [ + 0.9644, + 0.9604, + 0.9583, + 0.9407, + 0.1818 + ], + [ + 0.9586, + 0.9537, + 0.9477, + 0.9238, + 0.1817 + ], + [ + 0.9522, + 0.9458, + 0.9343, + 0.9032, + 0.1845 + ], + [ + 0.9418, + 0.9387, + 0.9236, + 0.8766, + 0.1849 + ], + [ + 0.9331, + 0.9297, + 0.9108, + 0.8358, + 0.1869 + ], + [ + 0.9215, + 0.9188, + 0.8927, + 0.2164, + 0.1904 + ], + [ + 0.9079, + 0.9053, + 0.8758, + 0.223, + 0.1927 + ], + [ + 0.8943, + 0.8882, + 0.8508, + 0.2275, + 0.1979 + ], + [ + 0.8761, + 0.8687, + 0.8142, + 0.2348, + 0.2025 + ] + ] +} \ No newline at end of file