diff --git a/.~lock.Poster.pptx# b/.~lock.Poster.pptx# deleted file mode 100644 index d987985..0000000 --- a/.~lock.Poster.pptx# +++ /dev/null @@ -1 +0,0 @@ -,sharpe,dhcp-150-250-90-28,28.04.2024 00:39,file:///home/sharpe/.var/app/org.libreoffice.LibreOffice/config/libreoffice/4; \ No newline at end of file diff --git a/20240428_12h59m15s_grim.png b/20240428_12h59m15s_grim.png new file mode 100644 index 0000000..03d751b Binary files /dev/null and b/20240428_12h59m15s_grim.png differ diff --git a/Filter_Analysis/CIFAR-10_FGSM_highest_rank.png b/Filter_Analysis/CIFAR-10_FGSM_highest_rank.png new file mode 100644 index 0000000..829625b Binary files /dev/null and b/Filter_Analysis/CIFAR-10_FGSM_highest_rank.png differ diff --git a/Filter_Analysis/CIFAR-10_FGSM_performance_map.png b/Filter_Analysis/CIFAR-10_FGSM_performance_map.png new file mode 100644 index 0000000..d45873a Binary files /dev/null and b/Filter_Analysis/CIFAR-10_FGSM_performance_map.png differ diff --git a/Filter_Analysis/MNIST_FGSM_highest_rank.png b/Filter_Analysis/MNIST_FGSM_highest_rank.png new file mode 100644 index 0000000..f999901 Binary files /dev/null and b/Filter_Analysis/MNIST_FGSM_highest_rank.png differ diff --git a/Filter_Analysis/MNIST_FGSM_performance_map.png b/Filter_Analysis/MNIST_FGSM_performance_map.png new file mode 100644 index 0000000..6c65550 Binary files /dev/null and b/Filter_Analysis/MNIST_FGSM_performance_map.png differ diff --git a/Filter_Analysis/] b/Filter_Analysis/] new file mode 100644 index 0000000..a661880 --- /dev/null +++ b/Filter_Analysis/] @@ -0,0 +1,59 @@ +import json +import numpy as np +import matplotlib.pyplot as plt +from matplotlib import cm +import copy + + + +def main(): + data = {} + with open("results/cifar10_fgsm.json", "r") as infile: + data = json.load(infile) + + attack = data["attack"] + epsilons = data["epsilons"] + filters = data["filters"] + dataset = data["dataset"] + strength_count = len(filters[list(filters.keys())[0]][0]) + + strengths = np.arange(strength_count) + epsilons = np.array(epsilons) + + # Assume constant step of strength and epsilon + dstrength = strengths[1] - strengths[0] + depsilon = epsilons[1] - epsilons[0] + + # Make a grid from strengths and epsilons + strengths, epsilons = np.meshgrid(strengths,epsilons) + #strengths, epsilons = strengths.ravel(), epsilons.ravel() + + colors = ('blue', 'orange', 'red', 'purple', 'green', 'yellow', 'brown') + + z = np.zeros_like(strengths) + best_performance = np.zeros_like(strengths) + bottoms = np.zeros_like(strengths) + + for i, filter in enumerate(filters): + performance = np.array(filters[filter]) + best_performance = np.where(performance > best_performance, i, best_performance) + z = np.where(performance > z, performance, z) + print(best_performance) + + for i, filter in enumerate(filters): + fig, ax = plt.subplots(subplot_kw={"projection": "3d"}) + tops = np.where(best_performance == i, best_performance, 0) + print(strengths.shape, epsilons.shape, bottoms.shape, dstrength.shape, depsilon.shape, tops.shape) + ax.bar3d(strengths, epsilons, bottoms, dstrength, depsilon, tops, color=colors[i]) + plt.show() + + ax.view_init(90, -90, 0) + + plt.legend() + plt.title(f"{filter} Performance") + plt.xlabel(f"{attack} Attack Strength ($\\epsilon$)") + plt.ylabel(f"{dataset} Classification Accuracy") + plt.show() + +if __name__ == "__main__": + main() diff --git a/Filter_Analysis/display_results.py b/Filter_Analysis/display_results.py index 15c6c05..17ce084 100644 --- a/Filter_Analysis/display_results.py +++ b/Filter_Analysis/display_results.py @@ -1,13 +1,18 @@ import json import numpy as np import matplotlib.pyplot as plt +from matplotlib import cm +from mpl_toolkits.mplot3d import Axes3D import copy def main(): + is_rank = False + has_random_guessing_threshold = False + data = {} - with open("results/mnist_fgsm.json", "r") as infile: + with open("results/cifar10_fgsm.json", "r") as infile: data = json.load(infile) attack = data["attack"] @@ -16,22 +21,73 @@ def main(): dataset = data["dataset"] strength_count = len(filters[list(filters.keys())[0]][0]) - for filter in filters: - plt.figure(figsize=(16,9)) - for i in range(strength_count): - filter_accuracy = [filters[filter][eps][i] for eps in range(len(epsilons))] - plt.plot(epsilons, filter_accuracy, label=f"Strength = {i}") + random_guess_threshold = np.ndarray((len(epsilons), strength_count)) + random_guess_threshold.fill(0.1) + print(random_guess_threshold.shape) + filters["Random Guess Threshold"] = random_guess_threshold - # Plot horizontal line at random guessing level - plt.hlines(0.1, epsilons[0], epsilons[-1], label="Random Guessing Threshold", colors="black", linestyles="dashed") + strengths = np.arange(strength_count) + epsilons = np.array(epsilons) - plt.legend(loc="upper right") - plt.title(f"{filter} Performance") - plt.xlabel(f"{attack} Attack Strength ($\\epsilon$)") - plt.ylabel(f"{dataset} Classification Accuracy") - plt.show() + # Assume constant step of strength and epsilon + dstrength = strengths[1] - strengths[0] + depsilon = epsilons[1] - epsilons[0] + # Make a grid from strengths and epsilons + strengths, epsilons = np.meshgrid(strengths,epsilons) + + colors = ('blue', 'orange', 'red', 'purple', 'green', 'yellow', 'brown', 'black') + + z = np.zeros_like(strengths) + best_performance = np.zeros_like(strengths) + + strengths, epsilons = strengths.ravel(), epsilons.ravel() + + for i, filter in enumerate(filters): + performance = np.array(filters[filter]) + best_performance = np.where(performance > z, i, best_performance) + z = np.where(performance > z, performance, z) + + z = z.ravel() + fig, ax = plt.subplots(subplot_kw={"projection": "3d"}, figsize=(9,10)) + for i, filter in enumerate(filters): + if i not in best_performance: + continue + tops = [] + x = [] + y = [] + for j, best in enumerate(best_performance.ravel()): + if best == i: + x.append(strengths[j]) + y.append(epsilons[j]) + tops.append(z[j]) + x = np.array(x) + y = np.array(y) + tops = np.array(tops) + greyscale = [((height+0.5)/1.5, 0, 0) if height > 0.1 else 'black' for height in tops] + if is_rank: + ax.bar3d(x, y, 0, dstrength, depsilon, tops, color=colors[i], zsort="average", shade=True, label=filter) + else: + if i < len(filters)-1: + ax.bar3d(x, y, 0, dstrength, depsilon, tops, color=greyscale, zsort="average", shade=True) + else: + ax.bar3d(x, y, 0, dstrength, depsilon, tops, color=greyscale, zsort="average", shade=True, label=filter) + has_random_guessing_threshold = True + + ax.zaxis.line.set_lw(0.) + ax.set_zticks([]) + ax.view_init(90, 0, 0) + ax.set_proj_type('ortho') + + if is_rank or has_random_guessing_threshold: + ax.legend(loc="lower center") + plt.title(f"{"Highest Rank Filters" if is_rank else "Filter Performance"} for {dataset}") + plt.ylabel(f"{attack} Attack Strength ($\\epsilon$)") + plt.xlabel("Filter Strength") + plt.clabel(f"{dataset} Classification Accuracy") + plt.savefig(f"{dataset}_{attack}_{"highest_rank" if is_rank else "performance_map"}.png") + #plt.show() if __name__ == "__main__": main() diff --git a/Poster.pptx b/Poster.pptx index 5fe5409..ad39ed4 100644 Binary files a/Poster.pptx and b/Poster.pptx differ