1-Bit, color snapping, and pluality vote accuracies
This commit is contained in:
Aidan Sharpe
parent
30d8766e69
commit
8f264f7958
Binary file not shown.
|
After Width: | Height: | Size: 163 KiB |
@ -4,6 +4,7 @@ import torch.nn.functional as F
|
||||
import torch.optim as optim
|
||||
from torchvision import datasets, transforms
|
||||
import numpy as np
|
||||
from scipy import stats
|
||||
import matplotlib.pyplot as plt
|
||||
import cv2
|
||||
from mnist import Net
|
||||
@ -69,12 +70,14 @@ def test(model, device, test_loader, epsilon):
|
||||
# Original dataset correct classifications
|
||||
orig_correct = 0
|
||||
# Attacked dataset correct classifications
|
||||
attacked_correct = 0
|
||||
unfiltered_correct = 0
|
||||
kuwahara_correct = 0
|
||||
bilateral_correct = 0
|
||||
gaussian_blur_correct = 0
|
||||
random_noise_correct = 0
|
||||
attacked_snap_color_correct = 0
|
||||
snap_color_correct = 0
|
||||
one_bit_correct = 0
|
||||
plurality_correct = 0
|
||||
|
||||
adv_examples = []
|
||||
|
||||
@ -111,30 +114,35 @@ def test(model, device, test_loader, epsilon):
|
||||
bilateral_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="bilateral")
|
||||
gaussian_blur_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="gaussian_blur")
|
||||
random_noise_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="noise")
|
||||
attacked_snap_color_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="snap_color")
|
||||
snap_color_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="snap_color")
|
||||
one_bit_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="1-bit")
|
||||
|
||||
# evaluate the model on the attacked and filtered images
|
||||
output_attacked = model(perturbed_data_normalized)
|
||||
output_unfiltered = model(perturbed_data_normalized)
|
||||
output_kuwahara = model(kuwahara_data)
|
||||
output_bilateral = model(bilateral_data)
|
||||
output_gaussian_blur = model(gaussian_blur_data)
|
||||
output_random_noise = model(random_noise_data)
|
||||
output_attacked_snap = model(attacked_snap_color_data)
|
||||
output_snap_color = model(snap_color_data)
|
||||
output_one_bit = model(one_bit_data)
|
||||
|
||||
# Get the predicted class from the model for each case
|
||||
attacked_pred = output_attacked.max(1, keepdim=True)[1]
|
||||
unfiltered_pred = output_unfiltered.max(1, keepdim=True)[1]
|
||||
kuwahara_pred = output_kuwahara.max(1, keepdim=True)[1]
|
||||
bilateral_pred = output_bilateral.max(1, keepdim=True)[1]
|
||||
gaussian_blur_pred = output_gaussian_blur.max(1, keepdim=True)[1]
|
||||
random_noise_pred = output_random_noise.max(1, keepdim=True)[1]
|
||||
attacked_snap_color_pred = output_attacked_snap.max(1, keepdim=True)[1]
|
||||
snap_color_pred = output_snap_color.max(1, keepdim=True)[1]
|
||||
one_bit_pred = output_one_bit.max(1, keepdim=True)[1]
|
||||
predictions = [unfiltered_pred.item(), kuwahara_pred.item(), bilateral_pred.item(), gaussian_blur_pred.item(), random_noise_pred.item(), snap_color_pred.item(), one_bit_pred.item()]
|
||||
plurality_pred = stats.mode(predictions)[0]
|
||||
|
||||
# Count up correct classifications for each case
|
||||
if orig_pred.item() == target.item():
|
||||
orig_correct += 1
|
||||
|
||||
if attacked_pred.item() == target.item():
|
||||
attacked_correct += 1
|
||||
if unfiltered_pred.item() == target.item():
|
||||
unfiltered_correct += 1
|
||||
|
||||
if kuwahara_pred.item() == target.item():
|
||||
kuwahara_correct += 1
|
||||
@ -148,28 +156,38 @@ def test(model, device, test_loader, epsilon):
|
||||
if random_noise_pred.item() == target.item():
|
||||
random_noise_correct += 1
|
||||
|
||||
if attacked_snap_color_pred.item() == target.item():
|
||||
attacked_snap_color_correct += 1
|
||||
if snap_color_pred.item() == target.item():
|
||||
snap_color_correct += 1
|
||||
|
||||
if one_bit_pred.item() == target.item():
|
||||
one_bit_correct += 1
|
||||
|
||||
if plurality_pred == target.item():
|
||||
plurality_correct += 1
|
||||
|
||||
# Calculate the overall accuracy of each case
|
||||
orig_acc = orig_correct/float(len(test_loader))
|
||||
attacked_acc = attacked_correct/float(len(test_loader))
|
||||
unfiltered_acc = unfiltered_correct/float(len(test_loader))
|
||||
kuwahara_acc = kuwahara_correct/float(len(test_loader))
|
||||
bilateral_acc = bilateral_correct/float(len(test_loader))
|
||||
gaussian_blur_acc = gaussian_blur_correct/float(len(test_loader))
|
||||
random_noise_acc = random_noise_correct/float(len(test_loader))
|
||||
attacked_snap_color_acc = attacked_snap_color_correct/float(len(test_loader))
|
||||
snap_color_acc = snap_color_correct/float(len(test_loader))
|
||||
one_bit_acc = one_bit_correct/float(len(test_loader))
|
||||
plurality_acc = plurality_correct/float(len(test_loader))
|
||||
|
||||
print(f"Epsilon: {epsilon}")
|
||||
print(f"====== EPSILON: {epsilon} ======")
|
||||
print(f"Clean (No Filter) Accuracy = {orig_correct} / {len(test_loader)} = {orig_acc}")
|
||||
print(f"Attacked (No Filter) Accuracy = {attacked_correct} / {len(test_loader)} = {attacked_acc}")
|
||||
print(f"Attacked (Kuwahara Filter) Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}")
|
||||
print(f"Attacked (Bilateral Filter) Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}")
|
||||
print(f"Attacked (Gaussian Blur) Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}")
|
||||
print(f"Attacked (Random Noise) Accuracy = {random_noise_correct} / {len(test_loader)} = {random_noise_acc}")
|
||||
print(f"Attacked (Snapped Color) Accuracy = {attacked_snap_color_correct} / {len(test_loader)} = {attacked_snap_color_acc}")
|
||||
print(f"Unfiltered Accuracy = {unfiltered_correct} / {len(test_loader)} = {unfiltered_acc}")
|
||||
print(f"Kuwahara Filter Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}")
|
||||
print(f"Bilateral Filter Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}")
|
||||
print(f"Gaussian Blur Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}")
|
||||
print(f"Random Noise Accuracy = {random_noise_correct} / {len(test_loader)} = {random_noise_acc}")
|
||||
print(f"Snapped Color Accuracy = {snap_color_correct} / {len(test_loader)} = {snap_color_acc}")
|
||||
print(f"1 Bit Accuracy = {one_bit_correct} / {len(test_loader)} = {one_bit_acc}")
|
||||
print(f"Plurality Vote Accuracy = {plurality_correct} / {len(test_loader)} = {plurality_acc}")
|
||||
|
||||
return attacked_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, attacked_snap_color_acc
|
||||
return unfiltered_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, snap_color_acc, one_bit_acc, plurality_acc
|
||||
|
||||
|
||||
def filtered(data, batch_size=64, filter="kuwahara"):
|
||||
@ -210,7 +228,7 @@ def filtered(data, batch_size=64, filter="kuwahara"):
|
||||
elif filter == "bilateral":
|
||||
for i in range(batch_size):
|
||||
filtered_images[i] = cv2.bilateralFilter(images[i], 5, 50, 50).reshape(filtered_images[i].shape)
|
||||
elif filter == "snap_color":
|
||||
elif filter == "1-bit":
|
||||
num_colors = 2
|
||||
for i in range(batch_size):
|
||||
# If the channel contains any negative values, define the lowest negative value as black
|
||||
@ -228,36 +246,45 @@ def filtered(data, batch_size=64, filter="kuwahara"):
|
||||
filtered_images[i] = filtered_images[i].astype(int).astype(float)/num_colors
|
||||
if (min_value < 0):
|
||||
filtered_images[i] -= min_value
|
||||
elif filter == "snap_color":
|
||||
for i in range(batch_size):
|
||||
filtered_images[i] = (images[i]*4).astype(int).astype(float) / 4
|
||||
|
||||
# Modify the data with the filtered image
|
||||
filtered_images = filtered_images.transpose(0,3,1,2)
|
||||
return torch.tensor(filtered_images).float()
|
||||
|
||||
attacked_accuracies = []
|
||||
unfiltered_accuracies = []
|
||||
kuwahara_accuracies = []
|
||||
bilateral_accuracies = []
|
||||
gaussian_blur_accuracies = []
|
||||
random_noise_accuracies = []
|
||||
attacked_snap_color_accuracies = []
|
||||
snap_color_accuracies = []
|
||||
one_bit_accuracies = []
|
||||
pluality_vote_accuracies = []
|
||||
|
||||
print(f"Model: {pretrained_model}")
|
||||
|
||||
for eps in epsilons:
|
||||
aacc, kacc, bacc, gacc, nacc, sacc = test(model, device, test_loader, eps)
|
||||
attacked_accuracies.append(aacc)
|
||||
kuwahara_accuracies.append(kacc)
|
||||
bilateral_accuracies.append(bacc)
|
||||
gaussian_blur_accuracies.append(gacc)
|
||||
random_noise_accuracies.append(nacc)
|
||||
attacked_snap_color_accuracies.append(sacc)
|
||||
unfiltered_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, snap_color_acc, one_bit_acc, plurality_acc = test(model, device, test_loader, eps)
|
||||
unfiltered_accuracies.append(unfiltered_acc)
|
||||
kuwahara_accuracies.append(kuwahara_acc)
|
||||
bilateral_accuracies.append(bilateral_acc)
|
||||
gaussian_blur_accuracies.append(gaussian_blur_acc)
|
||||
random_noise_accuracies.append(random_noise_acc)
|
||||
snap_color_accuracies.append(snap_color_acc)
|
||||
one_bit_accuracies.append(one_bit_acc)
|
||||
pluality_vote_accuracies.append(plurality_acc)
|
||||
|
||||
# Plot the results
|
||||
plt.plot(epsilons, attacked_accuracies, label="Attacked Accuracy")
|
||||
plt.plot(epsilons, unfiltered_accuracies, label="Attacked Accuracy")
|
||||
plt.plot(epsilons, kuwahara_accuracies, label="Kuwahara Accuracy")
|
||||
plt.plot(epsilons, bilateral_accuracies, label="Bilateral Accuracy")
|
||||
plt.plot(epsilons, gaussian_blur_accuracies, label="Gaussian Blur Accuracy")
|
||||
plt.plot(epsilons, random_noise_accuracies, label="Random Noise Accuracy")
|
||||
plt.plot(epsilons, attacked_snap_color_accuracies, label="Snapped Color Accuracy")
|
||||
plt.plot(epsilons, snap_color_accuracies, label="Snapped Color Accuracy")
|
||||
plt.plot(epsilons, one_bit_accuracies, label="1-Bit Accuracy")
|
||||
plt.plot(epsilons, pluality_vote_accuracies, label="Plurality Vote Accuracy")
|
||||
plt.legend()
|
||||
|
||||
plt.show()
|
||||
|
||||
@ -46,64 +46,87 @@
|
||||
## Models Defended with Various Filters
|
||||
|
||||
### Tabulated Results
|
||||
| $\epsilon$ | FGSM | Kuwahara | Bilateral | Gaussian Blur | Random Noise | Snapped Color |
|
||||
|------------|--------|----------|-----------|---------------|--------------|---------------|
|
||||
| 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | 0.9781 |
|
||||
| 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | 0.8818 |
|
||||
| 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | 0.8408 |
|
||||
| 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | 0.7496 |
|
||||
| 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | 0.4301 |
|
||||
| 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | 0.2091 |
|
||||
| $\epsilon$ | Unfiltered | Kuwahara | Bilateral | Gaussian Blur | Random Noise | Snapped Color | 1-Bit | Plurality |
|
||||
|------------|------------|----------|-----------|---------------|--------------|---------------|--------|-----------|
|
||||
| 0.00 | 0.992 | 0.9066 | 0.9391 | 0.9682 | 0.9911 | 0.9913 | 0.9722 | 0.9889 |
|
||||
| 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | 0.9781 | 0.8409 | 0.9600 |
|
||||
| 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | 0.8818 | 0.7919 | 0.8799 |
|
||||
| 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | 0.8408 | 0.7329 | 0.7879 |
|
||||
| 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | 0.7496 | 0.6794 | 0.6467 |
|
||||
| 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | 0.4301 | 0.6233 | 0.4721 |
|
||||
| 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | 0.2091 | 0. | |
|
||||
|
||||
### Plotted Results
|
||||
![]()
|
||||
|
||||
|
||||
### Raw Program Output
|
||||
Epsilon: 0.05
|
||||
====== EPSILON: 0.0 ======
|
||||
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
|
||||
Attacked (No Filter) Accuracy = 9600 / 10000 = 0.96
|
||||
Attacked (Kuwahara Filter) Accuracy = 8700 / 10000 = 0.87
|
||||
Attacked (Bilateral Filter) Accuracy = 8902 / 10000 = 0.8902
|
||||
Attacked (Gaussian Blur) Accuracy = 9271 / 10000 = 0.9271
|
||||
Attacked (Random Noise) Accuracy = 9603 / 10000 = 0.9603
|
||||
Attacked (Snapped Color) Accuracy = 9781 / 10000 = 0.9781
|
||||
Epsilon: 0.1
|
||||
Unfiltered Accuracy = 9920 / 10000 = 0.992
|
||||
Kuwahara Filter Accuracy = 9066 / 10000 = 0.9066
|
||||
Bilateral Filter Accuracy = 9391 / 10000 = 0.9391
|
||||
Gaussian Blur Accuracy = 9682 / 10000 = 0.9682
|
||||
Random Noise Accuracy = 9911 / 10000 = 0.9911
|
||||
Snapped Color Accuracy = 9913 / 10000 = 0.9913
|
||||
1 Bit Accuracy = 9722 / 10000 = 0.9722
|
||||
Plurality Vote Accuracy = 9889 / 10000 = 0.9889
|
||||
====== EPSILON: 0.05 ======
|
||||
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
|
||||
Attacked (No Filter) Accuracy = 8753 / 10000 = 0.8753
|
||||
Attacked (Kuwahara Filter) Accuracy = 8123 / 10000 = 0.8123
|
||||
Attacked (Bilateral Filter) Accuracy = 8133 / 10000 = 0.8133
|
||||
Attacked (Gaussian Blur) Accuracy = 8516 / 10000 = 0.8516
|
||||
Attacked (Random Noise) Accuracy = 8677 / 10000 = 0.8677
|
||||
Attacked (Snapped Color) Accuracy = 8818 / 10000 = 0.8818
|
||||
Epsilon: 0.15000000000000002
|
||||
Unfiltered Accuracy = 9600 / 10000 = 0.96
|
||||
Kuwahara Filter Accuracy = 8700 / 10000 = 0.87
|
||||
Bilateral Filter Accuracy = 8902 / 10000 = 0.8902
|
||||
Gaussian Blur Accuracy = 9271 / 10000 = 0.9271
|
||||
Random Noise Accuracy = 9587 / 10000 = 0.9587
|
||||
Snapped Color Accuracy = 9781 / 10000 = 0.9781
|
||||
1 Bit Accuracy = 8409 / 10000 = 0.8409
|
||||
Plurality Vote Accuracy = 9600 / 10000 = 0.96
|
||||
====== EPSILON: 0.1 ======
|
||||
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
|
||||
Attacked (No Filter) Accuracy = 7229 / 10000 = 0.7229
|
||||
Attacked (Kuwahara Filter) Accuracy = 7328 / 10000 = 0.7328
|
||||
Attacked (Bilateral Filter) Accuracy = 7098 / 10000 = 0.7098
|
||||
Attacked (Gaussian Blur) Accuracy = 7415 / 10000 = 0.7415
|
||||
Attacked (Random Noise) Accuracy = 7153 / 10000 = 0.7153
|
||||
Attacked (Snapped Color) Accuracy = 8408 / 10000 = 0.8408
|
||||
Epsilon: 0.2
|
||||
Unfiltered Accuracy = 8753 / 10000 = 0.8753
|
||||
Kuwahara Filter Accuracy = 8123 / 10000 = 0.8123
|
||||
Bilateral Filter Accuracy = 8133 / 10000 = 0.8133
|
||||
Gaussian Blur Accuracy = 8516 / 10000 = 0.8516
|
||||
Random Noise Accuracy = 8696 / 10000 = 0.8696
|
||||
Snapped Color Accuracy = 8818 / 10000 = 0.8818
|
||||
1 Bit Accuracy = 7919 / 10000 = 0.7919
|
||||
Plurality Vote Accuracy = 8799 / 10000 = 0.8799
|
||||
====== EPSILON: 0.15000000000000002 ======
|
||||
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
|
||||
Attacked (No Filter) Accuracy = 5008 / 10000 = 0.5008
|
||||
Attacked (Kuwahara Filter) Accuracy = 6301 / 10000 = 0.6301
|
||||
Attacked (Bilateral Filter) Accuracy = 5683 / 10000 = 0.5683
|
||||
Attacked (Gaussian Blur) Accuracy = 5983 / 10000 = 0.5983
|
||||
Attacked (Random Noise) Accuracy = 4941 / 10000 = 0.4941
|
||||
Attacked (Snapped Color) Accuracy = 7496 / 10000 = 0.7496
|
||||
Epsilon: 0.25
|
||||
Unfiltered Accuracy = 7229 / 10000 = 0.7229
|
||||
Kuwahara Filter Accuracy = 7328 / 10000 = 0.7328
|
||||
Bilateral Filter Accuracy = 7098 / 10000 = 0.7098
|
||||
Gaussian Blur Accuracy = 7415 / 10000 = 0.7415
|
||||
Random Noise Accuracy = 7119 / 10000 = 0.7119
|
||||
Snapped Color Accuracy = 8408 / 10000 = 0.8408
|
||||
1 Bit Accuracy = 7329 / 10000 = 0.7329
|
||||
Plurality Vote Accuracy = 7879 / 10000 = 0.7879
|
||||
====== EPSILON: 0.2 ======
|
||||
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
|
||||
Attacked (No Filter) Accuracy = 2922 / 10000 = 0.2922
|
||||
Attacked (Kuwahara Filter) Accuracy = 5197 / 10000 = 0.5197
|
||||
Attacked (Bilateral Filter) Accuracy = 4381 / 10000 = 0.4381
|
||||
Attacked (Gaussian Blur) Accuracy = 4591 / 10000 = 0.4591
|
||||
Attacked (Random Noise) Accuracy = 2843 / 10000 = 0.2843
|
||||
Attacked (Snapped Color) Accuracy = 4301 / 10000 = 0.4301
|
||||
Epsilon: 0.3
|
||||
Unfiltered Accuracy = 5008 / 10000 = 0.5008
|
||||
Kuwahara Filter Accuracy = 6301 / 10000 = 0.6301
|
||||
Bilateral Filter Accuracy = 5683 / 10000 = 0.5683
|
||||
Gaussian Blur Accuracy = 5983 / 10000 = 0.5983
|
||||
Random Noise Accuracy = 4933 / 10000 = 0.4933
|
||||
Snapped Color Accuracy = 7496 / 10000 = 0.7496
|
||||
1 Bit Accuracy = 6794 / 10000 = 0.6794
|
||||
Plurality Vote Accuracy = 6467 / 10000 = 0.6467
|
||||
====== EPSILON: 0.25 ======
|
||||
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
|
||||
Attacked (No Filter) Accuracy = 1599 / 10000 = 0.1599
|
||||
Attacked (Kuwahara Filter) Accuracy = 3981 / 10000 = 0.3981
|
||||
Attacked (Bilateral Filter) Accuracy = 3364 / 10000 = 0.3364
|
||||
Attacked (Gaussian Blur) Accuracy = 3481 / 10000 = 0.3481
|
||||
Attacked (Random Noise) Accuracy = 1584 / 10000 = 0.1584
|
||||
Attacked (Snapped Color) Accuracy = 2091 / 10000 = 0.2091
|
||||
Unfiltered Accuracy = 2922 / 10000 = 0.2922
|
||||
Kuwahara Filter Accuracy = 5197 / 10000 = 0.5197
|
||||
Bilateral Filter Accuracy = 4381 / 10000 = 0.4381
|
||||
Gaussian Blur Accuracy = 4591 / 10000 = 0.4591
|
||||
Random Noise Accuracy = 2876 / 10000 = 0.2876
|
||||
Snapped Color Accuracy = 4301 / 10000 = 0.4301
|
||||
1 Bit Accuracy = 6233 / 10000 = 0.6233
|
||||
Plurality Vote Accuracy = 4721 / 10000 = 0.4721
|
||||
====== EPSILON: 0.30000000000000004 ======
|
||||
Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
|
||||
Unfiltered Accuracy = 1599 / 10000 = 0.1599
|
||||
Kuwahara Filter Accuracy = 3981 / 10000 = 0.3981
|
||||
Bilateral Filter Accuracy = 3364 / 10000 = 0.3364
|
||||
Gaussian Blur Accuracy = 3481 / 10000 = 0.3481
|
||||
Random Noise Accuracy = 1560 / 10000 = 0.156
|
||||
Snapped Color Accuracy = 2091 / 10000 = 0.2091
|
||||
1 Bit Accuracy = 5462 / 10000 = 0.5462
|
||||
Plurality Vote Accuracy = 3312 / 10000 = 0.3312
|
||||
|
||||
Loading…
Reference in New Issue
Block a user