Adog64/Adversarial-Machine-Learning-Clinic
1
0
Fork 0
Code Issues 3 Pull Requests Packages Projects 1 Releases Wiki Activity

1-Bit, color snapping, and pluality vote accuracies

Browse Source
This commit is contained in:
Aidan Sharpe 2024-04-07 12:04:46 -04:00
parent 30d8766e69
commit 8f264f7958
3 changed files with 134 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
Filter_Analysis/Plurality_Vote_Accuracy_Agaisnt_Individual_Filters.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 163 KiB

93
Filter_Analysis/fgsm.py
View File

@ -4,6 +4,7 @@ import torch.nn.functional as F
import torch.optim as optim import torch.optim as optim
from torchvision import datasets, transforms from torchvision import datasets, transforms
import numpy as np import numpy as np
from scipy import stats
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import cv2 import cv2
from mnist import Net from mnist import Net
@ -69,12 +70,14 @@ def test(model, device, test_loader, epsilon):
# Original dataset correct classifications # Original dataset correct classifications
orig_correct = 0 orig_correct = 0
# Attacked dataset correct classifications # Attacked dataset correct classifications
attacked_correct = 0 unfiltered_correct = 0
kuwahara_correct = 0 kuwahara_correct = 0
bilateral_correct = 0 bilateral_correct = 0
gaussian_blur_correct = 0 gaussian_blur_correct = 0
random_noise_correct = 0 random_noise_correct = 0
attacked_snap_color_correct = 0 snap_color_correct = 0
one_bit_correct = 0
plurality_correct = 0
adv_examples = [] adv_examples = []
@ -111,30 +114,35 @@ def test(model, device, test_loader, epsilon):
bilateral_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="bilateral") bilateral_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="bilateral")
gaussian_blur_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="gaussian_blur") gaussian_blur_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="gaussian_blur")
random_noise_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="noise") random_noise_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="noise")
attacked_snap_color_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="snap_color") snap_color_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="snap_color")
one_bit_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="1-bit")
# evaluate the model on the attacked and filtered images # evaluate the model on the attacked and filtered images
output_attacked = model(perturbed_data_normalized) output_unfiltered = model(perturbed_data_normalized)
output_kuwahara = model(kuwahara_data) output_kuwahara = model(kuwahara_data)
output_bilateral = model(bilateral_data) output_bilateral = model(bilateral_data)
output_gaussian_blur = model(gaussian_blur_data) output_gaussian_blur = model(gaussian_blur_data)
output_random_noise = model(random_noise_data) output_random_noise = model(random_noise_data)
output_attacked_snap = model(attacked_snap_color_data) output_snap_color = model(snap_color_data)
output_one_bit = model(one_bit_data)
# Get the predicted class from the model for each case # Get the predicted class from the model for each case
attacked_pred = output_attacked.max(1, keepdim=True)[1] unfiltered_pred = output_unfiltered.max(1, keepdim=True)[1]
kuwahara_pred = output_kuwahara.max(1, keepdim=True)[1] kuwahara_pred = output_kuwahara.max(1, keepdim=True)[1]
bilateral_pred = output_bilateral.max(1, keepdim=True)[1] bilateral_pred = output_bilateral.max(1, keepdim=True)[1]
gaussian_blur_pred = output_gaussian_blur.max(1, keepdim=True)[1] gaussian_blur_pred = output_gaussian_blur.max(1, keepdim=True)[1]
random_noise_pred = output_random_noise.max(1, keepdim=True)[1] random_noise_pred = output_random_noise.max(1, keepdim=True)[1]
attacked_snap_color_pred = output_attacked_snap.max(1, keepdim=True)[1] snap_color_pred = output_snap_color.max(1, keepdim=True)[1]
one_bit_pred = output_one_bit.max(1, keepdim=True)[1]
predictions = [unfiltered_pred.item(), kuwahara_pred.item(), bilateral_pred.item(), gaussian_blur_pred.item(), random_noise_pred.item(), snap_color_pred.item(), one_bit_pred.item()]
plurality_pred = stats.mode(predictions)[0]
# Count up correct classifications for each case # Count up correct classifications for each case
if orig_pred.item() == target.item(): if orig_pred.item() == target.item():
orig_correct += 1 orig_correct += 1
if attacked_pred.item() == target.item(): if unfiltered_pred.item() == target.item():
attacked_correct += 1 unfiltered_correct += 1
if kuwahara_pred.item() == target.item(): if kuwahara_pred.item() == target.item():
kuwahara_correct += 1 kuwahara_correct += 1
@ -148,28 +156,38 @@ def test(model, device, test_loader, epsilon):
if random_noise_pred.item() == target.item(): if random_noise_pred.item() == target.item():
random_noise_correct += 1 random_noise_correct += 1
if attacked_snap_color_pred.item() == target.item(): if snap_color_pred.item() == target.item():
attacked_snap_color_correct += 1 snap_color_correct += 1
if one_bit_pred.item() == target.item():
one_bit_correct += 1
if plurality_pred == target.item():
plurality_correct += 1
# Calculate the overall accuracy of each case # Calculate the overall accuracy of each case
orig_acc = orig_correct/float(len(test_loader)) orig_acc = orig_correct/float(len(test_loader))
attacked_acc = attacked_correct/float(len(test_loader)) unfiltered_acc = unfiltered_correct/float(len(test_loader))
kuwahara_acc = kuwahara_correct/float(len(test_loader)) kuwahara_acc = kuwahara_correct/float(len(test_loader))
bilateral_acc = bilateral_correct/float(len(test_loader)) bilateral_acc = bilateral_correct/float(len(test_loader))
gaussian_blur_acc = gaussian_blur_correct/float(len(test_loader)) gaussian_blur_acc = gaussian_blur_correct/float(len(test_loader))
random_noise_acc = random_noise_correct/float(len(test_loader)) random_noise_acc = random_noise_correct/float(len(test_loader))
attacked_snap_color_acc = attacked_snap_color_correct/float(len(test_loader)) snap_color_acc = snap_color_correct/float(len(test_loader))
one_bit_acc = one_bit_correct/float(len(test_loader))
plurality_acc = plurality_correct/float(len(test_loader))
print(f"Epsilon: {epsilon}") print(f"====== EPSILON: {epsilon} ======")
print(f"Clean (No Filter) Accuracy = {orig_correct} / {len(test_loader)} = {orig_acc}") print(f"Clean (No Filter) Accuracy = {orig_correct} / {len(test_loader)} = {orig_acc}")
print(f"Attacked (No Filter) Accuracy = {attacked_correct} / {len(test_loader)} = {attacked_acc}") print(f"Unfiltered Accuracy = {unfiltered_correct} / {len(test_loader)} = {unfiltered_acc}")
print(f"Attacked (Kuwahara Filter) Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}") print(f"Kuwahara Filter Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}")
print(f"Attacked (Bilateral Filter) Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}") print(f"Bilateral Filter Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}")
print(f"Attacked (Gaussian Blur) Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}") print(f"Gaussian Blur Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}")
print(f"Attacked (Random Noise) Accuracy = {random_noise_correct} / {len(test_loader)} = {random_noise_acc}") print(f"Random Noise Accuracy = {random_noise_correct} / {len(test_loader)} = {random_noise_acc}")
print(f"Attacked (Snapped Color) Accuracy = {attacked_snap_color_correct} / {len(test_loader)} = {attacked_snap_color_acc}") print(f"Snapped Color Accuracy = {snap_color_correct} / {len(test_loader)} = {snap_color_acc}")
print(f"1 Bit Accuracy = {one_bit_correct} / {len(test_loader)} = {one_bit_acc}")
print(f"Plurality Vote Accuracy = {plurality_correct} / {len(test_loader)} = {plurality_acc}")
return attacked_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, attacked_snap_color_acc return unfiltered_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, snap_color_acc, one_bit_acc, plurality_acc
def filtered(data, batch_size=64, filter="kuwahara"): def filtered(data, batch_size=64, filter="kuwahara"):
@ -210,7 +228,7 @@ def filtered(data, batch_size=64, filter="kuwahara"):
elif filter == "bilateral": elif filter == "bilateral":
for i in range(batch_size): for i in range(batch_size):
filtered_images[i] = cv2.bilateralFilter(images[i], 5, 50, 50).reshape(filtered_images[i].shape) filtered_images[i] = cv2.bilateralFilter(images[i], 5, 50, 50).reshape(filtered_images[i].shape)
elif filter == "snap_color": elif filter == "1-bit":
num_colors = 2 num_colors = 2
for i in range(batch_size): for i in range(batch_size):
# If the channel contains any negative values, define the lowest negative value as black # If the channel contains any negative values, define the lowest negative value as black
@ -228,36 +246,45 @@ def filtered(data, batch_size=64, filter="kuwahara"):
filtered_images[i] = filtered_images[i].astype(int).astype(float)/num_colors filtered_images[i] = filtered_images[i].astype(int).astype(float)/num_colors
if (min_value < 0): if (min_value < 0):
filtered_images[i] -= min_value filtered_images[i] -= min_value
elif filter == "snap_color":
for i in range(batch_size):
filtered_images[i] = (images[i]*4).astype(int).astype(float) / 4
# Modify the data with the filtered image # Modify the data with the filtered image
filtered_images = filtered_images.transpose(0,3,1,2) filtered_images = filtered_images.transpose(0,3,1,2)
return torch.tensor(filtered_images).float() return torch.tensor(filtered_images).float()
attacked_accuracies = [] unfiltered_accuracies = []
kuwahara_accuracies = [] kuwahara_accuracies = []
bilateral_accuracies = [] bilateral_accuracies = []
gaussian_blur_accuracies = [] gaussian_blur_accuracies = []
random_noise_accuracies = [] random_noise_accuracies = []
attacked_snap_color_accuracies = [] snap_color_accuracies = []
one_bit_accuracies = []
pluality_vote_accuracies = []
print(f"Model: {pretrained_model}") print(f"Model: {pretrained_model}")
for eps in epsilons: for eps in epsilons:
aacc, kacc, bacc, gacc, nacc, sacc = test(model, device, test_loader, eps) unfiltered_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, snap_color_acc, one_bit_acc, plurality_acc = test(model, device, test_loader, eps)
attacked_accuracies.append(aacc) unfiltered_accuracies.append(unfiltered_acc)
kuwahara_accuracies.append(kacc) kuwahara_accuracies.append(kuwahara_acc)
bilateral_accuracies.append(bacc) bilateral_accuracies.append(bilateral_acc)
gaussian_blur_accuracies.append(gacc) gaussian_blur_accuracies.append(gaussian_blur_acc)
random_noise_accuracies.append(nacc) random_noise_accuracies.append(random_noise_acc)
attacked_snap_color_accuracies.append(sacc) snap_color_accuracies.append(snap_color_acc)
one_bit_accuracies.append(one_bit_acc)
pluality_vote_accuracies.append(plurality_acc)
# Plot the results # Plot the results
plt.plot(epsilons, attacked_accuracies, label="Attacked Accuracy") plt.plot(epsilons, unfiltered_accuracies, label="Attacked Accuracy")
plt.plot(epsilons, kuwahara_accuracies, label="Kuwahara Accuracy") plt.plot(epsilons, kuwahara_accuracies, label="Kuwahara Accuracy")
plt.plot(epsilons, bilateral_accuracies, label="Bilateral Accuracy") plt.plot(epsilons, bilateral_accuracies, label="Bilateral Accuracy")
plt.plot(epsilons, gaussian_blur_accuracies, label="Gaussian Blur Accuracy") plt.plot(epsilons, gaussian_blur_accuracies, label="Gaussian Blur Accuracy")
plt.plot(epsilons, random_noise_accuracies, label="Random Noise Accuracy") plt.plot(epsilons, random_noise_accuracies, label="Random Noise Accuracy")
plt.plot(epsilons, attacked_snap_color_accuracies, label="Snapped Color Accuracy") plt.plot(epsilons, snap_color_accuracies, label="Snapped Color Accuracy")
plt.plot(epsilons, one_bit_accuracies, label="1-Bit Accuracy")
plt.plot(epsilons, pluality_vote_accuracies, label="Plurality Vote Accuracy")
plt.legend() plt.legend()
plt.show() plt.show()

125
Filter_Analysis/wiki/Results.md
View File

@ -46,64 +46,87 @@
## Models Defended with Various Filters ## Models Defended with Various Filters
### Tabulated Results ### Tabulated Results
| $\epsilon$ | FGSM | Kuwahara | Bilateral | Gaussian Blur | Random Noise | Snapped Color | | $\epsilon$ | Unfiltered | Kuwahara | Bilateral | Gaussian Blur | Random Noise | Snapped Color | 1-Bit | Plurality |
|------------|--------|----------|-----------|---------------|--------------|---------------| |------------|------------|----------|-----------|---------------|--------------|---------------|--------|-----------|
| 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | 0.9781 | | 0.00 | 0.992 | 0.9066 | 0.9391 | 0.9682 | 0.9911 | 0.9913 | 0.9722 | 0.9889 |
| 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | 0.8818 | | 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | 0.9781 | 0.8409 | 0.9600 |
| 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | 0.8408 | | 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | 0.8818 | 0.7919 | 0.8799 |
| 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | 0.7496 | | 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | 0.8408 | 0.7329 | 0.7879 |
| 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | 0.4301 | | 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | 0.7496 | 0.6794 | 0.6467 |
| 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | 0.2091 | | 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | 0.4301 | 0.6233 | 0.4721 |
| 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | 0.2091 | 0. | |
### Plotted Results ### Plotted Results
![]() ![Results Plot](../Plurality_Vote_Accuracy_Agaisnt_Individual_Filters.png)
### Raw Program Output ### Raw Program Output
Epsilon: 0.05 ====== EPSILON: 0.0 ======
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked (No Filter) Accuracy = 9600 / 10000 = 0.96 Unfiltered Accuracy = 9920 / 10000 = 0.992
Attacked (Kuwahara Filter) Accuracy = 8700 / 10000 = 0.87 Kuwahara Filter Accuracy = 9066 / 10000 = 0.9066
Attacked (Bilateral Filter) Accuracy = 8902 / 10000 = 0.8902 Bilateral Filter Accuracy = 9391 / 10000 = 0.9391
Attacked (Gaussian Blur) Accuracy = 9271 / 10000 = 0.9271 Gaussian Blur Accuracy = 9682 / 10000 = 0.9682
Attacked (Random Noise) Accuracy = 9603 / 10000 = 0.9603 Random Noise Accuracy = 9911 / 10000 = 0.9911
Attacked (Snapped Color) Accuracy = 9781 / 10000 = 0.9781 Snapped Color Accuracy = 9913 / 10000 = 0.9913
Epsilon: 0.1 1 Bit Accuracy = 9722 / 10000 = 0.9722
Plurality Vote Accuracy = 9889 / 10000 = 0.9889
====== EPSILON: 0.05 ======
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked (No Filter) Accuracy = 8753 / 10000 = 0.8753 Unfiltered Accuracy = 9600 / 10000 = 0.96
Attacked (Kuwahara Filter) Accuracy = 8123 / 10000 = 0.8123 Kuwahara Filter Accuracy = 8700 / 10000 = 0.87
Attacked (Bilateral Filter) Accuracy = 8133 / 10000 = 0.8133 Bilateral Filter Accuracy = 8902 / 10000 = 0.8902
Attacked (Gaussian Blur) Accuracy = 8516 / 10000 = 0.8516 Gaussian Blur Accuracy = 9271 / 10000 = 0.9271
Attacked (Random Noise) Accuracy = 8677 / 10000 = 0.8677 Random Noise Accuracy = 9587 / 10000 = 0.9587
Attacked (Snapped Color) Accuracy = 8818 / 10000 = 0.8818 Snapped Color Accuracy = 9781 / 10000 = 0.9781
Epsilon: 0.15000000000000002 1 Bit Accuracy = 8409 / 10000 = 0.8409
Plurality Vote Accuracy = 9600 / 10000 = 0.96
====== EPSILON: 0.1 ======
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked (No Filter) Accuracy = 7229 / 10000 = 0.7229 Unfiltered Accuracy = 8753 / 10000 = 0.8753
Attacked (Kuwahara Filter) Accuracy = 7328 / 10000 = 0.7328 Kuwahara Filter Accuracy = 8123 / 10000 = 0.8123
Attacked (Bilateral Filter) Accuracy = 7098 / 10000 = 0.7098 Bilateral Filter Accuracy = 8133 / 10000 = 0.8133
Attacked (Gaussian Blur) Accuracy = 7415 / 10000 = 0.7415 Gaussian Blur Accuracy = 8516 / 10000 = 0.8516
Attacked (Random Noise) Accuracy = 7153 / 10000 = 0.7153 Random Noise Accuracy = 8696 / 10000 = 0.8696
Attacked (Snapped Color) Accuracy = 8408 / 10000 = 0.8408 Snapped Color Accuracy = 8818 / 10000 = 0.8818
Epsilon: 0.2 1 Bit Accuracy = 7919 / 10000 = 0.7919
Plurality Vote Accuracy = 8799 / 10000 = 0.8799
====== EPSILON: 0.15000000000000002 ======
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked (No Filter) Accuracy = 5008 / 10000 = 0.5008 Unfiltered Accuracy = 7229 / 10000 = 0.7229
Attacked (Kuwahara Filter) Accuracy = 6301 / 10000 = 0.6301 Kuwahara Filter Accuracy = 7328 / 10000 = 0.7328
Attacked (Bilateral Filter) Accuracy = 5683 / 10000 = 0.5683 Bilateral Filter Accuracy = 7098 / 10000 = 0.7098
Attacked (Gaussian Blur) Accuracy = 5983 / 10000 = 0.5983 Gaussian Blur Accuracy = 7415 / 10000 = 0.7415
Attacked (Random Noise) Accuracy = 4941 / 10000 = 0.4941 Random Noise Accuracy = 7119 / 10000 = 0.7119
Attacked (Snapped Color) Accuracy = 7496 / 10000 = 0.7496 Snapped Color Accuracy = 8408 / 10000 = 0.8408
Epsilon: 0.25 1 Bit Accuracy = 7329 / 10000 = 0.7329
Plurality Vote Accuracy = 7879 / 10000 = 0.7879
====== EPSILON: 0.2 ======
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked (No Filter) Accuracy = 2922 / 10000 = 0.2922 Unfiltered Accuracy = 5008 / 10000 = 0.5008
Attacked (Kuwahara Filter) Accuracy = 5197 / 10000 = 0.5197 Kuwahara Filter Accuracy = 6301 / 10000 = 0.6301
Attacked (Bilateral Filter) Accuracy = 4381 / 10000 = 0.4381 Bilateral Filter Accuracy = 5683 / 10000 = 0.5683
Attacked (Gaussian Blur) Accuracy = 4591 / 10000 = 0.4591 Gaussian Blur Accuracy = 5983 / 10000 = 0.5983
Attacked (Random Noise) Accuracy = 2843 / 10000 = 0.2843 Random Noise Accuracy = 4933 / 10000 = 0.4933
Attacked (Snapped Color) Accuracy = 4301 / 10000 = 0.4301 Snapped Color Accuracy = 7496 / 10000 = 0.7496
Epsilon: 0.3 1 Bit Accuracy = 6794 / 10000 = 0.6794
Plurality Vote Accuracy = 6467 / 10000 = 0.6467
====== EPSILON: 0.25 ======
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked (No Filter) Accuracy = 1599 / 10000 = 0.1599 Unfiltered Accuracy = 2922 / 10000 = 0.2922
Attacked (Kuwahara Filter) Accuracy = 3981 / 10000 = 0.3981 Kuwahara Filter Accuracy = 5197 / 10000 = 0.5197
Attacked (Bilateral Filter) Accuracy = 3364 / 10000 = 0.3364 Bilateral Filter Accuracy = 4381 / 10000 = 0.4381
Attacked (Gaussian Blur) Accuracy = 3481 / 10000 = 0.3481 Gaussian Blur Accuracy = 4591 / 10000 = 0.4591
Attacked (Random Noise) Accuracy = 1584 / 10000 = 0.1584 Random Noise Accuracy = 2876 / 10000 = 0.2876
Attacked (Snapped Color) Accuracy = 2091 / 10000 = 0.2091 Snapped Color Accuracy = 4301 / 10000 = 0.4301
1 Bit Accuracy = 6233 / 10000 = 0.6233
Plurality Vote Accuracy = 4721 / 10000 = 0.4721
====== EPSILON: 0.30000000000000004 ======
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Unfiltered Accuracy = 1599 / 10000 = 0.1599
Kuwahara Filter Accuracy = 3981 / 10000 = 0.3981
Bilateral Filter Accuracy = 3364 / 10000 = 0.3364
Gaussian Blur Accuracy = 3481 / 10000 = 0.3481
Random Noise Accuracy = 1560 / 10000 = 0.156
Snapped Color Accuracy = 2091 / 10000 = 0.2091
1 Bit Accuracy = 5462 / 10000 = 0.5462
Plurality Vote Accuracy = 3312 / 10000 = 0.3312