diff --git a/Filter_Analysis/Plurality_Vote_Accuracy_Agaisnt_Individual_Filters.png b/Filter_Analysis/Plurality_Vote_Accuracy_Agaisnt_Individual_Filters.png new file mode 100644 index 0000000..dd844f4 Binary files /dev/null and b/Filter_Analysis/Plurality_Vote_Accuracy_Agaisnt_Individual_Filters.png differ diff --git a/Filter_Analysis/fgsm.py b/Filter_Analysis/fgsm.py index cf1dc54..50b32aa 100644 --- a/Filter_Analysis/fgsm.py +++ b/Filter_Analysis/fgsm.py @@ -4,6 +4,7 @@ import torch.nn.functional as F import torch.optim as optim from torchvision import datasets, transforms import numpy as np +from scipy import stats import matplotlib.pyplot as plt import cv2 from mnist import Net @@ -69,12 +70,14 @@ def test(model, device, test_loader, epsilon): # Original dataset correct classifications orig_correct = 0 # Attacked dataset correct classifications - attacked_correct = 0 + unfiltered_correct = 0 kuwahara_correct = 0 bilateral_correct = 0 gaussian_blur_correct = 0 random_noise_correct = 0 - attacked_snap_color_correct = 0 + snap_color_correct = 0 + one_bit_correct = 0 + plurality_correct = 0 adv_examples = [] @@ -111,30 +114,35 @@ def test(model, device, test_loader, epsilon): bilateral_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="bilateral") gaussian_blur_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="gaussian_blur") random_noise_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="noise") - attacked_snap_color_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="snap_color") + snap_color_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="snap_color") + one_bit_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="1-bit") # evaluate the model on the attacked and filtered images - output_attacked = model(perturbed_data_normalized) + output_unfiltered = model(perturbed_data_normalized) output_kuwahara = model(kuwahara_data) output_bilateral = model(bilateral_data) output_gaussian_blur = model(gaussian_blur_data) output_random_noise = model(random_noise_data) - output_attacked_snap = model(attacked_snap_color_data) + output_snap_color = model(snap_color_data) + output_one_bit = model(one_bit_data) # Get the predicted class from the model for each case - attacked_pred = output_attacked.max(1, keepdim=True)[1] + unfiltered_pred = output_unfiltered.max(1, keepdim=True)[1] kuwahara_pred = output_kuwahara.max(1, keepdim=True)[1] bilateral_pred = output_bilateral.max(1, keepdim=True)[1] gaussian_blur_pred = output_gaussian_blur.max(1, keepdim=True)[1] random_noise_pred = output_random_noise.max(1, keepdim=True)[1] - attacked_snap_color_pred = output_attacked_snap.max(1, keepdim=True)[1] + snap_color_pred = output_snap_color.max(1, keepdim=True)[1] + one_bit_pred = output_one_bit.max(1, keepdim=True)[1] + predictions = [unfiltered_pred.item(), kuwahara_pred.item(), bilateral_pred.item(), gaussian_blur_pred.item(), random_noise_pred.item(), snap_color_pred.item(), one_bit_pred.item()] + plurality_pred = stats.mode(predictions)[0] # Count up correct classifications for each case if orig_pred.item() == target.item(): orig_correct += 1 - if attacked_pred.item() == target.item(): - attacked_correct += 1 + if unfiltered_pred.item() == target.item(): + unfiltered_correct += 1 if kuwahara_pred.item() == target.item(): kuwahara_correct += 1 @@ -148,28 +156,38 @@ def test(model, device, test_loader, epsilon): if random_noise_pred.item() == target.item(): random_noise_correct += 1 - if attacked_snap_color_pred.item() == target.item(): - attacked_snap_color_correct += 1 + if snap_color_pred.item() == target.item(): + snap_color_correct += 1 + + if one_bit_pred.item() == target.item(): + one_bit_correct += 1 + + if plurality_pred == target.item(): + plurality_correct += 1 # Calculate the overall accuracy of each case orig_acc = orig_correct/float(len(test_loader)) - attacked_acc = attacked_correct/float(len(test_loader)) + unfiltered_acc = unfiltered_correct/float(len(test_loader)) kuwahara_acc = kuwahara_correct/float(len(test_loader)) bilateral_acc = bilateral_correct/float(len(test_loader)) gaussian_blur_acc = gaussian_blur_correct/float(len(test_loader)) random_noise_acc = random_noise_correct/float(len(test_loader)) - attacked_snap_color_acc = attacked_snap_color_correct/float(len(test_loader)) + snap_color_acc = snap_color_correct/float(len(test_loader)) + one_bit_acc = one_bit_correct/float(len(test_loader)) + plurality_acc = plurality_correct/float(len(test_loader)) - print(f"Epsilon: {epsilon}") + print(f"====== EPSILON: {epsilon} ======") print(f"Clean (No Filter) Accuracy = {orig_correct} / {len(test_loader)} = {orig_acc}") - print(f"Attacked (No Filter) Accuracy = {attacked_correct} / {len(test_loader)} = {attacked_acc}") - print(f"Attacked (Kuwahara Filter) Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}") - print(f"Attacked (Bilateral Filter) Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}") - print(f"Attacked (Gaussian Blur) Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}") - print(f"Attacked (Random Noise) Accuracy = {random_noise_correct} / {len(test_loader)} = {random_noise_acc}") - print(f"Attacked (Snapped Color) Accuracy = {attacked_snap_color_correct} / {len(test_loader)} = {attacked_snap_color_acc}") + print(f"Unfiltered Accuracy = {unfiltered_correct} / {len(test_loader)} = {unfiltered_acc}") + print(f"Kuwahara Filter Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}") + print(f"Bilateral Filter Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}") + print(f"Gaussian Blur Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}") + print(f"Random Noise Accuracy = {random_noise_correct} / {len(test_loader)} = {random_noise_acc}") + print(f"Snapped Color Accuracy = {snap_color_correct} / {len(test_loader)} = {snap_color_acc}") + print(f"1 Bit Accuracy = {one_bit_correct} / {len(test_loader)} = {one_bit_acc}") + print(f"Plurality Vote Accuracy = {plurality_correct} / {len(test_loader)} = {plurality_acc}") - return attacked_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, attacked_snap_color_acc + return unfiltered_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, snap_color_acc, one_bit_acc, plurality_acc def filtered(data, batch_size=64, filter="kuwahara"): @@ -210,7 +228,7 @@ def filtered(data, batch_size=64, filter="kuwahara"): elif filter == "bilateral": for i in range(batch_size): filtered_images[i] = cv2.bilateralFilter(images[i], 5, 50, 50).reshape(filtered_images[i].shape) - elif filter == "snap_color": + elif filter == "1-bit": num_colors = 2 for i in range(batch_size): # If the channel contains any negative values, define the lowest negative value as black @@ -228,36 +246,45 @@ def filtered(data, batch_size=64, filter="kuwahara"): filtered_images[i] = filtered_images[i].astype(int).astype(float)/num_colors if (min_value < 0): filtered_images[i] -= min_value + elif filter == "snap_color": + for i in range(batch_size): + filtered_images[i] = (images[i]*4).astype(int).astype(float) / 4 # Modify the data with the filtered image filtered_images = filtered_images.transpose(0,3,1,2) return torch.tensor(filtered_images).float() -attacked_accuracies = [] +unfiltered_accuracies = [] kuwahara_accuracies = [] bilateral_accuracies = [] gaussian_blur_accuracies = [] random_noise_accuracies = [] -attacked_snap_color_accuracies = [] +snap_color_accuracies = [] +one_bit_accuracies = [] +pluality_vote_accuracies = [] print(f"Model: {pretrained_model}") for eps in epsilons: - aacc, kacc, bacc, gacc, nacc, sacc = test(model, device, test_loader, eps) - attacked_accuracies.append(aacc) - kuwahara_accuracies.append(kacc) - bilateral_accuracies.append(bacc) - gaussian_blur_accuracies.append(gacc) - random_noise_accuracies.append(nacc) - attacked_snap_color_accuracies.append(sacc) + unfiltered_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, snap_color_acc, one_bit_acc, plurality_acc = test(model, device, test_loader, eps) + unfiltered_accuracies.append(unfiltered_acc) + kuwahara_accuracies.append(kuwahara_acc) + bilateral_accuracies.append(bilateral_acc) + gaussian_blur_accuracies.append(gaussian_blur_acc) + random_noise_accuracies.append(random_noise_acc) + snap_color_accuracies.append(snap_color_acc) + one_bit_accuracies.append(one_bit_acc) + pluality_vote_accuracies.append(plurality_acc) # Plot the results -plt.plot(epsilons, attacked_accuracies, label="Attacked Accuracy") +plt.plot(epsilons, unfiltered_accuracies, label="Attacked Accuracy") plt.plot(epsilons, kuwahara_accuracies, label="Kuwahara Accuracy") plt.plot(epsilons, bilateral_accuracies, label="Bilateral Accuracy") plt.plot(epsilons, gaussian_blur_accuracies, label="Gaussian Blur Accuracy") plt.plot(epsilons, random_noise_accuracies, label="Random Noise Accuracy") -plt.plot(epsilons, attacked_snap_color_accuracies, label="Snapped Color Accuracy") +plt.plot(epsilons, snap_color_accuracies, label="Snapped Color Accuracy") +plt.plot(epsilons, one_bit_accuracies, label="1-Bit Accuracy") +plt.plot(epsilons, pluality_vote_accuracies, label="Plurality Vote Accuracy") plt.legend() plt.show() diff --git a/Filter_Analysis/wiki/Results.md b/Filter_Analysis/wiki/Results.md index 3912a84..5c82c6d 100644 --- a/Filter_Analysis/wiki/Results.md +++ b/Filter_Analysis/wiki/Results.md @@ -46,64 +46,87 @@ ## Models Defended with Various Filters ### Tabulated Results -| $\epsilon$ | FGSM | Kuwahara | Bilateral | Gaussian Blur | Random Noise | Snapped Color | -|------------|--------|----------|-----------|---------------|--------------|---------------| -| 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | 0.9781 | -| 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | 0.8818 | -| 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | 0.8408 | -| 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | 0.7496 | -| 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | 0.4301 | -| 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | 0.2091 | +| $\epsilon$ | Unfiltered | Kuwahara | Bilateral | Gaussian Blur | Random Noise | Snapped Color | 1-Bit | Plurality | +|------------|------------|----------|-----------|---------------|--------------|---------------|--------|-----------| +| 0.00 | 0.992 | 0.9066 | 0.9391 | 0.9682 | 0.9911 | 0.9913 | 0.9722 | 0.9889 | +| 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | 0.9781 | 0.8409 | 0.9600 | +| 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | 0.8818 | 0.7919 | 0.8799 | +| 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | 0.8408 | 0.7329 | 0.7879 | +| 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | 0.7496 | 0.6794 | 0.6467 | +| 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | 0.4301 | 0.6233 | 0.4721 | +| 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | 0.2091 | 0. | | ### Plotted Results -![]() +![Results Plot](../Plurality_Vote_Accuracy_Agaisnt_Individual_Filters.png) ### Raw Program Output -Epsilon: 0.05 +====== EPSILON: 0.0 ====== Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 -Attacked (No Filter) Accuracy = 9600 / 10000 = 0.96 -Attacked (Kuwahara Filter) Accuracy = 8700 / 10000 = 0.87 -Attacked (Bilateral Filter) Accuracy = 8902 / 10000 = 0.8902 -Attacked (Gaussian Blur) Accuracy = 9271 / 10000 = 0.9271 -Attacked (Random Noise) Accuracy = 9603 / 10000 = 0.9603 -Attacked (Snapped Color) Accuracy = 9781 / 10000 = 0.9781 -Epsilon: 0.1 +Unfiltered Accuracy = 9920 / 10000 = 0.992 +Kuwahara Filter Accuracy = 9066 / 10000 = 0.9066 +Bilateral Filter Accuracy = 9391 / 10000 = 0.9391 +Gaussian Blur Accuracy = 9682 / 10000 = 0.9682 +Random Noise Accuracy = 9911 / 10000 = 0.9911 +Snapped Color Accuracy = 9913 / 10000 = 0.9913 +1 Bit Accuracy = 9722 / 10000 = 0.9722 +Plurality Vote Accuracy = 9889 / 10000 = 0.9889 +====== EPSILON: 0.05 ====== Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 -Attacked (No Filter) Accuracy = 8753 / 10000 = 0.8753 -Attacked (Kuwahara Filter) Accuracy = 8123 / 10000 = 0.8123 -Attacked (Bilateral Filter) Accuracy = 8133 / 10000 = 0.8133 -Attacked (Gaussian Blur) Accuracy = 8516 / 10000 = 0.8516 -Attacked (Random Noise) Accuracy = 8677 / 10000 = 0.8677 -Attacked (Snapped Color) Accuracy = 8818 / 10000 = 0.8818 -Epsilon: 0.15000000000000002 +Unfiltered Accuracy = 9600 / 10000 = 0.96 +Kuwahara Filter Accuracy = 8700 / 10000 = 0.87 +Bilateral Filter Accuracy = 8902 / 10000 = 0.8902 +Gaussian Blur Accuracy = 9271 / 10000 = 0.9271 +Random Noise Accuracy = 9587 / 10000 = 0.9587 +Snapped Color Accuracy = 9781 / 10000 = 0.9781 +1 Bit Accuracy = 8409 / 10000 = 0.8409 +Plurality Vote Accuracy = 9600 / 10000 = 0.96 +====== EPSILON: 0.1 ====== Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 -Attacked (No Filter) Accuracy = 7229 / 10000 = 0.7229 -Attacked (Kuwahara Filter) Accuracy = 7328 / 10000 = 0.7328 -Attacked (Bilateral Filter) Accuracy = 7098 / 10000 = 0.7098 -Attacked (Gaussian Blur) Accuracy = 7415 / 10000 = 0.7415 -Attacked (Random Noise) Accuracy = 7153 / 10000 = 0.7153 -Attacked (Snapped Color) Accuracy = 8408 / 10000 = 0.8408 -Epsilon: 0.2 +Unfiltered Accuracy = 8753 / 10000 = 0.8753 +Kuwahara Filter Accuracy = 8123 / 10000 = 0.8123 +Bilateral Filter Accuracy = 8133 / 10000 = 0.8133 +Gaussian Blur Accuracy = 8516 / 10000 = 0.8516 +Random Noise Accuracy = 8696 / 10000 = 0.8696 +Snapped Color Accuracy = 8818 / 10000 = 0.8818 +1 Bit Accuracy = 7919 / 10000 = 0.7919 +Plurality Vote Accuracy = 8799 / 10000 = 0.8799 +====== EPSILON: 0.15000000000000002 ====== Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 -Attacked (No Filter) Accuracy = 5008 / 10000 = 0.5008 -Attacked (Kuwahara Filter) Accuracy = 6301 / 10000 = 0.6301 -Attacked (Bilateral Filter) Accuracy = 5683 / 10000 = 0.5683 -Attacked (Gaussian Blur) Accuracy = 5983 / 10000 = 0.5983 -Attacked (Random Noise) Accuracy = 4941 / 10000 = 0.4941 -Attacked (Snapped Color) Accuracy = 7496 / 10000 = 0.7496 -Epsilon: 0.25 +Unfiltered Accuracy = 7229 / 10000 = 0.7229 +Kuwahara Filter Accuracy = 7328 / 10000 = 0.7328 +Bilateral Filter Accuracy = 7098 / 10000 = 0.7098 +Gaussian Blur Accuracy = 7415 / 10000 = 0.7415 +Random Noise Accuracy = 7119 / 10000 = 0.7119 +Snapped Color Accuracy = 8408 / 10000 = 0.8408 +1 Bit Accuracy = 7329 / 10000 = 0.7329 +Plurality Vote Accuracy = 7879 / 10000 = 0.7879 +====== EPSILON: 0.2 ====== Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 -Attacked (No Filter) Accuracy = 2922 / 10000 = 0.2922 -Attacked (Kuwahara Filter) Accuracy = 5197 / 10000 = 0.5197 -Attacked (Bilateral Filter) Accuracy = 4381 / 10000 = 0.4381 -Attacked (Gaussian Blur) Accuracy = 4591 / 10000 = 0.4591 -Attacked (Random Noise) Accuracy = 2843 / 10000 = 0.2843 -Attacked (Snapped Color) Accuracy = 4301 / 10000 = 0.4301 -Epsilon: 0.3 +Unfiltered Accuracy = 5008 / 10000 = 0.5008 +Kuwahara Filter Accuracy = 6301 / 10000 = 0.6301 +Bilateral Filter Accuracy = 5683 / 10000 = 0.5683 +Gaussian Blur Accuracy = 5983 / 10000 = 0.5983 +Random Noise Accuracy = 4933 / 10000 = 0.4933 +Snapped Color Accuracy = 7496 / 10000 = 0.7496 +1 Bit Accuracy = 6794 / 10000 = 0.6794 +Plurality Vote Accuracy = 6467 / 10000 = 0.6467 +====== EPSILON: 0.25 ====== Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 -Attacked (No Filter) Accuracy = 1599 / 10000 = 0.1599 -Attacked (Kuwahara Filter) Accuracy = 3981 / 10000 = 0.3981 -Attacked (Bilateral Filter) Accuracy = 3364 / 10000 = 0.3364 -Attacked (Gaussian Blur) Accuracy = 3481 / 10000 = 0.3481 -Attacked (Random Noise) Accuracy = 1584 / 10000 = 0.1584 -Attacked (Snapped Color) Accuracy = 2091 / 10000 = 0.2091 +Unfiltered Accuracy = 2922 / 10000 = 0.2922 +Kuwahara Filter Accuracy = 5197 / 10000 = 0.5197 +Bilateral Filter Accuracy = 4381 / 10000 = 0.4381 +Gaussian Blur Accuracy = 4591 / 10000 = 0.4591 +Random Noise Accuracy = 2876 / 10000 = 0.2876 +Snapped Color Accuracy = 4301 / 10000 = 0.4301 +1 Bit Accuracy = 6233 / 10000 = 0.6233 +Plurality Vote Accuracy = 4721 / 10000 = 0.4721 +====== EPSILON: 0.30000000000000004 ====== +Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 +Unfiltered Accuracy = 1599 / 10000 = 0.1599 +Kuwahara Filter Accuracy = 3981 / 10000 = 0.3981 +Bilateral Filter Accuracy = 3364 / 10000 = 0.3364 +Gaussian Blur Accuracy = 3481 / 10000 = 0.3481 +Random Noise Accuracy = 1560 / 10000 = 0.156 +Snapped Color Accuracy = 2091 / 10000 = 0.2091 +1 Bit Accuracy = 5462 / 10000 = 0.5462 +Plurality Vote Accuracy = 3312 / 10000 = 0.3312