diff --git a/Filter_Analysis/Filter_Performance_Against_FGSM_Attack.png b/Filter_Analysis/Filter_Performance_Against_FGSM_Attack.png new file mode 100644 index 0000000..ce43d7a Binary files /dev/null and b/Filter_Analysis/Filter_Performance_Against_FGSM_Attack.png differ diff --git a/Filter_Analysis/Filter_Performance_Against_FGSM_Attack_With_Snapped.png b/Filter_Analysis/Filter_Performance_Against_FGSM_Attack_With_Snapped.png new file mode 100644 index 0000000..aa8f832 Binary files /dev/null and b/Filter_Analysis/Filter_Performance_Against_FGSM_Attack_With_Snapped.png differ diff --git a/Filter_Analysis/fgsm.py b/Filter_Analysis/fgsm.py index d3750fc..75a82d2 100644 --- a/Filter_Analysis/fgsm.py +++ b/Filter_Analysis/fgsm.py @@ -73,7 +73,8 @@ def test(model, device, test_loader, epsilon): kuwahara_correct = 0 bilateral_correct = 0 gaussian_blur_correct = 0 - noisy_correct = 0 + random_noise_correct = 0 + attacked_snap_color_correct = 0 adv_examples = [] @@ -109,55 +110,66 @@ def test(model, device, test_loader, epsilon): kuwahara_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="kuwahara") bilateral_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="bilateral") gaussian_blur_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="gaussian_blur") - noisy_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="noise") + random_noise_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="noise") + attacked_snap_color_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="snap_color") # evaluate the model on the attacked and filtered images output_attacked = model(perturbed_data_normalized) output_kuwahara = model(kuwahara_data) output_bilateral = model(bilateral_data) output_gaussian_blur = model(gaussian_blur_data) - output_noisy = model(noisy_data) + output_random_noise = model(random_noise_data) + output_attacked_snap = model(attacked_snap_color_data) + # Get the predicted class from the model for each case attacked_pred = output_attacked.max(1, keepdim=True)[1] kuwahara_pred = output_kuwahara.max(1, keepdim=True)[1] bilateral_pred = output_bilateral.max(1, keepdim=True)[1] gaussian_blur_pred = output_gaussian_blur.max(1, keepdim=True)[1] - noisy_pred = output_noisy.max(1, keepdim=True)[1] + random_noise_pred = output_random_noise.max(1, keepdim=True)[1] + attacked_snap_color_pred = output_attacked_snap.max(1, keepdim=True)[1] + # Count up correct classifications for each case if orig_pred.item() == target.item(): orig_correct += 1 if attacked_pred.item() == target.item(): attacked_correct += 1 - + if kuwahara_pred.item() == target.item(): kuwahara_correct += 1 if bilateral_pred.item() == target.item(): bilateral_correct += 1 - + if gaussian_blur_pred.item() == target.item(): gaussian_blur_correct += 1 - if noisy_pred.item() == target.item(): - noisy_correct += 1 + if random_noise_pred.item() == target.item(): + random_noise_correct += 1 + if attacked_snap_color_pred.item() == target.item(): + attacked_snap_color_correct += 1 + + # Calculate the overall accuracy of each case orig_acc = orig_correct/float(len(test_loader)) attacked_acc = attacked_correct/float(len(test_loader)) kuwahara_acc = kuwahara_correct/float(len(test_loader)) bilateral_acc = bilateral_correct/float(len(test_loader)) gaussian_blur_acc = gaussian_blur_correct/float(len(test_loader)) - noisy_acc = noisy_correct/float(len(test_loader)) + random_noise_acc = random_noise_correct/float(len(test_loader)) + attacked_snap_color_acc = attacked_snap_color_correct/float(len(test_loader)) print(f"Epsilon: {epsilon}") - print(f"Original Accuracy = {orig_correct} / {len(test_loader)} = {orig_acc}") - print(f"Attacked Accuracy = {attacked_correct} / {len(test_loader)} = {attacked_acc}") - print(f"Kuwahara Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}") - print(f"Bilateral Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}") - print(f"Gaussian Blur Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}") - print(f"Noisy Accuracy = {noisy_correct} / {len(test_loader)} = {noisy_acc}") + print(f"Clean (No Filter) Accuracy = {orig_correct} / {len(test_loader)} = {orig_acc}") + print(f"Attacked (No Filter) Accuracy = {attacked_correct} / {len(test_loader)} = {attacked_acc}") + print(f"Attacked (Kuwahara Filter) Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}") + print(f"Attacked (Bilateral Filter) Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}") + print(f"Attacked (Gaussian Blur) Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}") + print(f"Attacked (Random Noise) Accuracy = {random_noise_correct} / {len(test_loader)} = {random_noise_acc}") + print(f"Attacked (Snapped Color) Accuracy = {attacked_snap_color_correct} / {len(test_loader)} = {attacked_snap_color_acc}") - return attacked_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, noisy_acc + return attacked_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, attacked_snap_color_acc def filtered(data, batch_size=64, filter="kuwahara"): @@ -167,6 +179,7 @@ def filtered(data, batch_size=64, filter="kuwahara"): images = data.numpy().transpose(0,2,3,1) except RuntimeError: images = data.detach().numpy().transpose(0,2,3,1) + # Apply the Kuwahara filter filtered_images = np.ndarray((batch_size,28,28,1)) @@ -197,6 +210,9 @@ def filtered(data, batch_size=64, filter="kuwahara"): elif filter == "bilateral": for i in range(batch_size): filtered_images[i] = cv2.bilateralFilter(images[i], 5, 50, 50).reshape(filtered_images[i].shape) + elif filter == "snap_color": + for i in range(batch_size): + filtered_images[i] = (images[i]*4).astype(int).astype(float) / 4 # Modify the data with the filtered image filtered_images = filtered_images.transpose(0,3,1,2) @@ -206,21 +222,27 @@ attacked_accuracies = [] kuwahara_accuracies = [] bilateral_accuracies = [] gaussian_blur_accuracies = [] -noisy_accuracies = [] +random_noise_accuracies = [] +attacked_snap_color_accuracies = [] print(f"Model: {pretrained_model}") for eps in epsilons: - aacc, kacc, bacc, gacc, nacc = test(model, device, test_loader, eps) + aacc, kacc, bacc, gacc, nacc, sacc = test(model, device, test_loader, eps) attacked_accuracies.append(aacc) kuwahara_accuracies.append(kacc) bilateral_accuracies.append(bacc) gaussian_blur_accuracies.append(gacc) - noisy_accuracies.append(nacc) + random_noise_accuracies.append(nacc) + attacked_snap_color_accuracies.append(sacc) # Plot the results plt.plot(epsilons, attacked_accuracies, label="Attacked Accuracy") plt.plot(epsilons, kuwahara_accuracies, label="Kuwahara Accuracy") +plt.plot(epsilons, bilateral_accuracies, label="Bilateral Accuracy") +plt.plot(epsilons, gaussian_blur_accuracies, label="Gaussian Blur Accuracy") +plt.plot(epsilons, random_noise_accuracies, label="Random Noise Accuracy") +plt.plot(epsilons, attacked_snap_color_accuracies, label="Snapped Color Accuracy") plt.legend() plt.show() diff --git a/Filter_Analysis/wiki/Results.md b/Filter_Analysis/wiki/Results.md index 535a6ce..3912a84 100644 --- a/Filter_Analysis/wiki/Results.md +++ b/Filter_Analysis/wiki/Results.md @@ -46,55 +46,64 @@ ## Models Defended with Various Filters ### Tabulated Results -| $\epsilon$ | FGSM | Kuwahara | Bilateral | Gaussian Blur | Random Noise | -|------------|--------|----------|-----------|---------------|--------------| -| 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | -| 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | -| 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | -| 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | -| 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | -| 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | +| $\epsilon$ | FGSM | Kuwahara | Bilateral | Gaussian Blur | Random Noise | Snapped Color | +|------------|--------|----------|-----------|---------------|--------------|---------------| +| 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | 0.9781 | +| 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | 0.8818 | +| 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | 0.8408 | +| 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | 0.7496 | +| 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | 0.4301 | +| 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | 0.2091 | + +### Plotted Results +![]() ### Raw Program Output Epsilon: 0.05 -Original Accuracy = 9920 / 10000 = 0.992 -Attacked Accuracy = 9600 / 10000 = 0.96 -Kuwahara Accuracy = 8700 / 10000 = 0.87 -Bilateral Accuracy = 8902 / 10000 = 0.8902 -Gaussian Blur Accuracy = 9271 / 10000 = 0.9271 -Noisy Accuracy = 9603 / 10000 = 0.9603 +Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 +Attacked (No Filter) Accuracy = 9600 / 10000 = 0.96 +Attacked (Kuwahara Filter) Accuracy = 8700 / 10000 = 0.87 +Attacked (Bilateral Filter) Accuracy = 8902 / 10000 = 0.8902 +Attacked (Gaussian Blur) Accuracy = 9271 / 10000 = 0.9271 +Attacked (Random Noise) Accuracy = 9603 / 10000 = 0.9603 +Attacked (Snapped Color) Accuracy = 9781 / 10000 = 0.9781 Epsilon: 0.1 -Original Accuracy = 9920 / 10000 = 0.992 -Attacked Accuracy = 8753 / 10000 = 0.8753 -Kuwahara Accuracy = 8123 / 10000 = 0.8123 -Bilateral Accuracy = 8133 / 10000 = 0.8133 -Gaussian Blur Accuracy = 8516 / 10000 = 0.8516 -Noisy Accuracy = 8677 / 10000 = 0.8677 +Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 +Attacked (No Filter) Accuracy = 8753 / 10000 = 0.8753 +Attacked (Kuwahara Filter) Accuracy = 8123 / 10000 = 0.8123 +Attacked (Bilateral Filter) Accuracy = 8133 / 10000 = 0.8133 +Attacked (Gaussian Blur) Accuracy = 8516 / 10000 = 0.8516 +Attacked (Random Noise) Accuracy = 8677 / 10000 = 0.8677 +Attacked (Snapped Color) Accuracy = 8818 / 10000 = 0.8818 Epsilon: 0.15000000000000002 -Original Accuracy = 9920 / 10000 = 0.992 -Attacked Accuracy = 7229 / 10000 = 0.7229 -Kuwahara Accuracy = 7328 / 10000 = 0.7328 -Bilateral Accuracy = 7098 / 10000 = 0.7098 -Gaussian Blur Accuracy = 7415 / 10000 = 0.7415 -Noisy Accuracy = 7153 / 10000 = 0.7153 +Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 +Attacked (No Filter) Accuracy = 7229 / 10000 = 0.7229 +Attacked (Kuwahara Filter) Accuracy = 7328 / 10000 = 0.7328 +Attacked (Bilateral Filter) Accuracy = 7098 / 10000 = 0.7098 +Attacked (Gaussian Blur) Accuracy = 7415 / 10000 = 0.7415 +Attacked (Random Noise) Accuracy = 7153 / 10000 = 0.7153 +Attacked (Snapped Color) Accuracy = 8408 / 10000 = 0.8408 Epsilon: 0.2 -Original Accuracy = 9920 / 10000 = 0.992 -Attacked Accuracy = 5008 / 10000 = 0.5008 -Kuwahara Accuracy = 6301 / 10000 = 0.6301 -Bilateral Accuracy = 5683 / 10000 = 0.5683 -Gaussian Blur Accuracy = 5983 / 10000 = 0.5983 -Noisy Accuracy = 4941 / 10000 = 0.4941 +Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 +Attacked (No Filter) Accuracy = 5008 / 10000 = 0.5008 +Attacked (Kuwahara Filter) Accuracy = 6301 / 10000 = 0.6301 +Attacked (Bilateral Filter) Accuracy = 5683 / 10000 = 0.5683 +Attacked (Gaussian Blur) Accuracy = 5983 / 10000 = 0.5983 +Attacked (Random Noise) Accuracy = 4941 / 10000 = 0.4941 +Attacked (Snapped Color) Accuracy = 7496 / 10000 = 0.7496 Epsilon: 0.25 -Original Accuracy = 9920 / 10000 = 0.992 -Attacked Accuracy = 2922 / 10000 = 0.2922 -Kuwahara Accuracy = 5197 / 10000 = 0.5197 -Bilateral Accuracy = 4381 / 10000 = 0.4381 -Gaussian Blur Accuracy = 4591 / 10000 = 0.4591 -Noisy Accuracy = 2843 / 10000 = 0.2843 +Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 +Attacked (No Filter) Accuracy = 2922 / 10000 = 0.2922 +Attacked (Kuwahara Filter) Accuracy = 5197 / 10000 = 0.5197 +Attacked (Bilateral Filter) Accuracy = 4381 / 10000 = 0.4381 +Attacked (Gaussian Blur) Accuracy = 4591 / 10000 = 0.4591 +Attacked (Random Noise) Accuracy = 2843 / 10000 = 0.2843 +Attacked (Snapped Color) Accuracy = 4301 / 10000 = 0.4301 Epsilon: 0.3 -Original Accuracy = 9920 / 10000 = 0.992 -Attacked Accuracy = 1599 / 10000 = 0.1599 -Kuwahara Accuracy = 3981 / 10000 = 0.3981 -Bilateral Accuracy = 3364 / 10000 = 0.3364 -Gaussian Blur Accuracy = 3481 / 10000 = 0.3481 -Noisy Accuracy = 1584 / 10000 = 0.1584 +Clean (No Filter) Accuracy = 9920 / 10000 = 0.992 +Attacked (No Filter) Accuracy = 1599 / 10000 = 0.1599 +Attacked (Kuwahara Filter) Accuracy = 3981 / 10000 = 0.3981 +Attacked (Bilateral Filter) Accuracy = 3364 / 10000 = 0.3364 +Attacked (Gaussian Blur) Accuracy = 3481 / 10000 = 0.3481 +Attacked (Random Noise) Accuracy = 1584 / 10000 = 0.1584 +Attacked (Snapped Color) Accuracy = 2091 / 10000 = 0.2091