Adog64/Adversarial-Machine-Learning-Clinic
1
0
Fork 0
Code Issues 3 Pull Requests Packages Projects 1 Releases Wiki Activity

Results successfully displayed as matplotlib bar3d

Browse Source
This commit is contained in:
Aidan Sharpe
2024-04-30 23:02:14 -04:00
parent b473ddc8e9
commit bf2440ef86
12 changed files with 772 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.~lock.Poster.pptx# Normal file
View File

@ -0,0 +1 @@
,sharpe,dhcp-150-250-208-169,30.04.2024 21:44,file:///home/sharpe/.var/app/org.libreoffice.LibreOffice/config/libreoffice/4;

BIN
Concept.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 48 KiB

BIN
Concept.xcf Normal file
View File

Binary file not shown.

BIN
Filter_Analysis/CIFAR-10_FGSM_3d.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 168 KiB

BIN
Filter_Analysis/MNIST_FGSM_3d.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 164 KiB

BIN
Filter_Analysis/__pycache__/mplot_3d_helper.cpython-311.pyc Normal file
View File

Binary file not shown.

102
Filter_Analysis/display_results.py
View File

@ -1,14 +1,14 @@
import json import json
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from matplotlib import cm from mplot_3d_helper import *
from mpl_toolkits.mplot3d import Axes3D from matplotlib.lines import Line2D
import copy import copy
def main(): def main():
is_rank = False is_rank = True
has_random_guessing_threshold = False has_random_guessing_threshold = False
data = {} data = {}
@ -37,7 +37,7 @@ def main():
# Make a grid from strengths and epsilons # Make a grid from strengths and epsilons
strengths, epsilons = np.meshgrid(strengths,epsilons) strengths, epsilons = np.meshgrid(strengths,epsilons)
colors = ('blue', 'orange', 'red', 'purple', 'green', 'yellow', 'brown', 'black') colors = ['blue', 'orange', 'red', 'purple', 'green', 'yellow', 'brown', 'black']
z = np.zeros_like(strengths) z = np.zeros_like(strengths)
best_performance = np.zeros_like(strengths) best_performance = np.zeros_like(strengths)
@ -50,44 +50,68 @@ def main():
z = np.where(performance > z, performance, z) z = np.where(performance > z, performance, z)
z = z.ravel() z = z.ravel()
fig, ax = plt.subplots(subplot_kw={"projection": "3d"}, figsize=(9,10)) #for i, filter in enumerate(filters):
for i, filter in enumerate(filters): # if i not in best_performance:
if i not in best_performance: # continue
continue # tops = []
tops = [] # x = []
x = [] # y = []
y = [] # for j, best in enumerate(best_performance.ravel()):
for j, best in enumerate(best_performance.ravel()): # if best == i:
if best == i: # x.append(strengths[j])
x.append(strengths[j]) # y.append(epsilons[j])
y.append(epsilons[j]) # tops.append(z[j])
tops.append(z[j]) # x = np.array(x)
x = np.array(x) # y = np.array(y)
y = np.array(y) # tops = np.array(tops)
tops = np.array(tops) # greyscale = [((height+0.5)/1.5, 0, 0) if height > 0.1 else 'black' for height in tops]
greyscale = [((height+0.5)/1.5, 0, 0) if height > 0.1 else 'black' for height in tops] # if is_rank:
if is_rank: # ax.bar3d(x, y, 0, dstrength, depsilon, tops, color=colors[i], zsort="max", shade=True, label=filter)
ax.bar3d(x, y, 0, dstrength, depsilon, tops, color=colors[i], zsort="average", shade=True, label=filter) # else:
else: # if i < len(filters)-1:
if i < len(filters)-1: # ax.bar3d(x, y, 0, dstrength, depsilon, tops, color=greyscale, zsort="average", shade=True)
ax.bar3d(x, y, 0, dstrength, depsilon, tops, color=greyscale, zsort="average", shade=True) # else:
else: # ax.bar3d(x, y, 0, dstrength, depsilon, tops, color=greyscale, zsort="average", shade=True, label=filter)
ax.bar3d(x, y, 0, dstrength, depsilon, tops, color=greyscale, zsort="average", shade=True, label=filter) # has_random_guessing_threshold = True
has_random_guessing_threshold = True
ax.zaxis.line.set_lw(0.) fig, ax = plt.subplots(subplot_kw={"projection": "3d"}, figsize=(9,10))
ax.set_zticks([]) ax.view_init(45, 135, 0)
ax.view_init(90, 0, 0)
ax.set_proj_type('ortho') ax.set_proj_type('ortho')
if is_rank or has_random_guessing_threshold: ax_x, ax_y, ax_z = sph_2_cart(*sph_view(ax))
ax.legend(loc="lower center") camera = np.array((ax_x, ax_y,0))
plt.title(f"{"Highest Rank Filters" if is_rank else "Filter Performance"} for {dataset}") z_order = get_distances(camera, strengths, epsilons, z)
plt.ylabel(f"{attack} Attack Strength ($\\epsilon$)")
plt.xlabel("Filter Strength") max_height = max(z_order)
plt.clabel(f"{dataset} Classification Accuracy") best_performance = best_performance.ravel()
plt.savefig(f"{dataset}_{attack}_{"highest_rank" if is_rank else "performance_map"}.png") for i, filter_idx in enumerate(best_performance):
#plt.show() filter = list(filters.keys())[filter_idx]
top = z[i]
x = strengths[i]
y = epsilons[i]
pl = ax.bar3d(x,y,0,dstrength,depsilon,top, color=colors[filter_idx], zsort="max")
pl._sort_zpos = max_height - z_order[i]
#ax.zaxis.line.set_lw(0.)
#ax.set_zticks([])
#if is_rank or has_random_guessing_threshold:
#ax.legend(loc="lower center")
#score_type = "Highest Rank Filters" if is_rank else "Filter Performance"
custom_lines = []
for color in colors:
custom_lines.append(Line2D([0], [0], color=color, lw=4))
filters = [filter.replace('_', ' ').title() for filter in filters]
fig.legend(custom_lines, filters, loc="outside lower right")
plt.title(f"Filter Efficacy for {dataset}", fontsize=40)
plt.ylabel(f"{attack} Attack Strength ($\\epsilon$)", fontsize=18)
plt.xlabel("Filter Strength", fontsize=18)
ax.set_zlabel(f"{dataset} Classification Accuracy", fontsize=18)
#score_type = "highest_rank" if is_rank else "performance_map"
plt.savefig(f"{dataset}_{attack}_3d.png")
plt.show()
if __name__ == "__main__": if __name__ == "__main__":
main() main()

25
Filter_Analysis/mplot_3d_helper.py Normal file
View File

@ -0,0 +1,25 @@
import numpy as np
import matplotlib.pyplot as plt
def sph_2_cart(r, theta, phi):
'''spherical to Cartesian transformation.'''
x = r * np.sin(theta) * np.cos(phi)
y = r * np.sin(theta) * np.sin(phi)
z = r * np.cos(theta)
return x, y, z
def sph_view(ax):
'''returns the camera position for 3D axes in spherical coordinates'''
r = np.square(np.max([ax.get_xlim(), ax.get_ylim()], 1)).sum()
theta, phi = np.radians((90-ax.elev, ax.azim))
return r, theta, phi
#
# end of apodemus's code
def get_distances(view, xpos, ypos, dz):
distances = []
a = np.array((xpos, ypos, dz))
for i in range(len(xpos)):
distance = (a[0, i] - view[0])**2 + (a[1, i] - view[1])**2 + (a[2, i] - view[2])**2
distances.append(np.sqrt(distance))
return distances

21
Filter_Analysis/reformat_data.py
View File

@ -1,26 +1,27 @@
import json import json
import numpy as np import numpy as np
import copy
def reformat_data(): def reformat_data():
results = {} results = {}
with open("results/mnist_fgsm.json", "r") as infile: with open("results/mnist_fgsm.json", "r") as infile:
results = json.load(infile) results = json.load(infile)
reformatted_results = {} #reformatted_results = {}
reformatted_results["dataset"] = "MNIST" #reformatted_results["dataset"] = "MNIST"
reformatted_results["attack"] = "FGSM" #reformatted_results["attack"] = "FGSM"
MAX_EPSILON = 0.3 #MAX_EPSILON = 0.3
EPSILON_STEP = 0.025 #EPSILON_STEP = 0.025
epsilons = np.arange(0.0, MAX_EPSILON+EPSILON_STEP, EPSILON_STEP) #epsilons = np.arange(0.0, MAX_EPSILON+EPSILON_STEP, EPSILON_STEP)
reformatted_results["epsilons"] = list(epsilons) #reformatted_results["epsilons"] = list(epsilons)
reformatted_results["filters"] = {} reformatted_results = copy.deepcopy(results)
filters = list(results.keys())[1:] filters = list(results.keys())[1:]
for filter in filters: for i, epsilon in enumerate(results["epsilons"]):
reformatted_results["filters"][filter] = results[filter] reformatted_results["filters"]["bit_depth"][i] = results["filters"]["bit_depth"][i][::-1]
reformatted_results_json = json.dumps(reformatted_results, indent=4) reformatted_results_json = json.dumps(reformatted_results, indent=4)
with open("results/mnist_fgsm_reformatted.json", "w") as outfile: with open("results/mnist_fgsm_reformatted.json", "w") as outfile:

672
Filter_Analysis/results/mnist_fgsm_reformatted.json Normal file
View File

@ -0,0 +1,672 @@
{
"dataset": "MNIST",
"attack": "FGSM",
"epsilons": [
0.0,
0.025,
0.05,
0.07500000000000001,
0.1,
0.125,
0.15000000000000002,
0.17500000000000002,
0.2,
0.225,
0.25,
0.275,
0.30000000000000004
],
"filters": {
"gaussian_blur": [
[
0.992,
0.9879,
0.9682,
0.7731,
0.525
],
[
0.9796,
0.9801,
0.9512,
0.7381,
0.4862
],
[
0.96,
0.9674,
0.9271,
0.6922,
0.4446
],
[
0.926,
0.946,
0.8939,
0.6427,
0.3989
],
[
0.8753,
0.9133,
0.8516,
0.5881,
0.3603
],
[
0.8104,
0.869,
0.7989,
0.5278,
0.3263
],
[
0.7229,
0.8135,
0.7415,
0.471,
0.2968
],
[
0.6207,
0.7456,
0.6741,
0.4224,
0.2683
],
[
0.5008,
0.6636,
0.5983,
0.3755,
0.2453
],
[
0.3894,
0.5821,
0.5243,
0.3359,
0.2269
],
[
0.2922,
0.505,
0.4591,
0.3034,
0.2112
],
[
0.2149,
0.429,
0.3998,
0.2743,
0.1983
],
[
0.1599,
0.3648,
0.3481,
0.2493,
0.1884
]
],
"gaussian_kuwahara": [
[
0.9897,
0.9766,
0.9066,
0.7355,
0.5131
],
[
0.9808,
0.9667,
0.8909,
0.7035,
0.4824
],
[
0.9651,
0.9547,
0.87,
0.6713,
0.4538
],
[
0.9412,
0.9334,
0.8447,
0.6354,
0.426
],
[
0.9035,
0.9107,
0.8123,
0.597,
0.3915
],
[
0.8539,
0.8785,
0.7751,
0.5616,
0.362
],
[
0.7925,
0.8328,
0.7328,
0.5236,
0.3344
],
[
0.7078,
0.7808,
0.6816,
0.4868,
0.309
],
[
0.6125,
0.7179,
0.6301,
0.4513,
0.2865
],
[
0.4979,
0.646,
0.5773,
0.4242,
0.2702
],
[
0.3927,
0.564,
0.5197,
0.3859,
0.2493
],
[
0.3023,
0.4761,
0.4594,
0.3494,
0.2354
],
[
0.2289,
0.3839,
0.3981,
0.3182,
0.2232
]
],
"mean_kuwahara": [
[
0.988,
0.7536,
0.3667,
0.1763,
0.1339
],
[
0.9795,
0.7359,
0.3496,
0.171,
0.1318
],
[
0.965,
0.7129,
0.3295,
0.1637,
0.1286
],
[
0.946,
0.6871,
0.3119,
0.1578,
0.1244
],
[
0.916,
0.6617,
0.2841,
0.1497,
0.1228
],
[
0.8746,
0.6317,
0.2587,
0.1422,
0.1211
],
[
0.8235,
0.6019,
0.2395,
0.136,
0.1193
],
[
0.7499,
0.5699,
0.2253,
0.134,
0.1164
],
[
0.665,
0.542,
0.2168,
0.1335,
0.1138
],
[
0.5642,
0.5087,
0.2064,
0.1328,
0.1129
],
[
0.4739,
0.4773,
0.1993,
0.1306,
0.1145
],
[
0.3638,
0.437,
0.1921,
0.1309,
0.1159
],
[
0.2659,
0.3912,
0.1854,
0.1307,
0.1166
]
],
"random_noise": [
[
0.9913,
0.9899,
0.9872,
0.9719,
0.9226
],
[
0.9793,
0.9782,
0.9711,
0.9453,
0.8802
],
[
0.9603,
0.9568,
0.9436,
0.9049,
0.8184
],
[
0.9253,
0.9183,
0.895,
0.8392,
0.7432
],
[
0.8743,
0.8653,
0.8309,
0.7656,
0.6606
],
[
0.809,
0.7948,
0.7486,
0.6709,
0.5588
],
[
0.721,
0.6999,
0.6485,
0.5625,
0.4577
],
[
0.6157,
0.5881,
0.5377,
0.4548,
0.3647
],
[
0.5005,
0.4802,
0.4267,
0.3571,
0.2885
],
[
0.385,
0.3668,
0.3295,
0.2696,
0.2223
],
[
0.2918,
0.2758,
0.244,
0.2039,
0.1724
],
[
0.215,
0.2016,
0.1832,
0.1555,
0.1326
],
[
0.1591,
0.154,
0.1371,
0.1163,
0.1021
]
],
"bilateral_filter": [
[
0.9887,
0.9887,
0.9391,
0.5584,
0.2568
],
[
0.9809,
0.9809,
0.9184,
0.5198,
0.241
],
[
0.9695,
0.9695,
0.8902,
0.4831,
0.2245
],
[
0.9482,
0.9482,
0.8533,
0.4436,
0.2079
],
[
0.9142,
0.9142,
0.8133,
0.4019,
0.1915
],
[
0.8714,
0.8714,
0.7656,
0.3641,
0.1792
],
[
0.8169,
0.8169,
0.7098,
0.3299,
0.1681
],
[
0.7477,
0.7477,
0.641,
0.2978,
0.161
],
[
0.6619,
0.6619,
0.5683,
0.2723,
0.1563
],
[
0.5767,
0.5767,
0.5003,
0.2476,
0.1517
],
[
0.4922,
0.4922,
0.4381,
0.2288,
0.1484
],
[
0.4133,
0.4133,
0.3836,
0.2126,
0.146
],
[
0.3468,
0.3468,
0.3364,
0.1999,
0.1444
]
],
"bit_depth": [
[
0.992,
0.992,
0.9916,
0.9913,
0.9894
],
[
0.9796,
0.9796,
0.9807,
0.9823,
0.9862
],
[
0.96,
0.9604,
0.965,
0.9781,
0.9808
],
[
0.926,
0.926,
0.9219,
0.9228,
0.9744
],
[
0.8753,
0.8751,
0.8747,
0.8818,
0.9424
],
[
0.8104,
0.8094,
0.821,
0.8621,
0.9307
],
[
0.7229,
0.7235,
0.7427,
0.8408,
0.9157
],
[
0.6207,
0.6229,
0.6554,
0.7794,
0.8972
],
[
0.5008,
0.5046,
0.559,
0.7496,
0.8799
],
[
0.3894,
0.3914,
0.4547,
0.5289,
0.8581
],
[
0.2922,
0.2927,
0.3113,
0.4301,
0.7603
],
[
0.2149,
0.2168,
0.2414,
0.3992,
0.7227
],
[
0.1599,
0.161,
0.1874,
0.2091,
0.2624
]
],
"threshold_filter": [
[
0.982,
0.9817,
0.9799,
0.9713,
0.9502
],
[
0.978,
0.9755,
0.9751,
0.9655,
0.9334
],
[
0.9728,
0.9713,
0.9696,
0.9578,
0.9077
],
[
0.9678,
0.9668,
0.9645,
0.9508,
0.1837
],
[
0.9644,
0.9604,
0.9583,
0.9407,
0.1818
],
[
0.9586,
0.9537,
0.9477,
0.9238,
0.1817
],
[
0.9522,
0.9458,
0.9343,
0.9032,
0.1845
],
[
0.9418,
0.9387,
0.9236,
0.8766,
0.1849
],
[
0.9331,
0.9297,
0.9108,
0.8358,
0.1869
],
[
0.9215,
0.9188,
0.8927,
0.2164,
0.1904
],
[
0.9079,
0.9053,
0.8758,
0.223,
0.1927
],
[
0.8943,
0.8882,
0.8508,
0.2275,
0.1979
],
[
0.8761,
0.8687,
0.8142,
0.2348,
0.2025
]
]
}
}

BIN
Poster.pptx
View File

Binary file not shown.

BIN
TheStrengthOfWeakLearnability.pdf Normal file
View File

Binary file not shown.