Adog64/Adversarial-Machine-Learning-Clinic
1
0
Fork 0
Code Issues 3 Pull Requests Packages Projects 1 Releases Wiki Activity

Implemented reduced color space (snapped color) filter

Browse Source
This commit is contained in:
Aidan Sharpe 2024-04-05 17:18:25 -04:00
parent 47b14362de
commit afd810a802
4 changed files with 94 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
Filter_Analysis/Filter_Performance_Against_FGSM_Attack.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 125 KiB

BIN
Filter_Analysis/Filter_Performance_Against_FGSM_Attack_With_Snapped.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 132 KiB

56
Filter_Analysis/fgsm.py
View File

@ -73,7 +73,8 @@ def test(model, device, test_loader, epsilon):
kuwahara_correct = 0 kuwahara_correct = 0
bilateral_correct = 0 bilateral_correct = 0
gaussian_blur_correct = 0 gaussian_blur_correct = 0
noisy_correct = 0 random_noise_correct = 0
attacked_snap_color_correct = 0
adv_examples = [] adv_examples = []
@ -109,21 +110,26 @@ def test(model, device, test_loader, epsilon):
kuwahara_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="kuwahara") kuwahara_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="kuwahara")
bilateral_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="bilateral") bilateral_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="bilateral")
gaussian_blur_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="gaussian_blur") gaussian_blur_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="gaussian_blur")
noisy_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="noise") random_noise_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="noise")
attacked_snap_color_data = filtered(perturbed_data_normalized, len(perturbed_data_normalized), filter="snap_color")
# evaluate the model on the attacked and filtered images # evaluate the model on the attacked and filtered images
output_attacked = model(perturbed_data_normalized) output_attacked = model(perturbed_data_normalized)
output_kuwahara = model(kuwahara_data) output_kuwahara = model(kuwahara_data)
output_bilateral = model(bilateral_data) output_bilateral = model(bilateral_data)
output_gaussian_blur = model(gaussian_blur_data) output_gaussian_blur = model(gaussian_blur_data)
output_noisy = model(noisy_data) output_random_noise = model(random_noise_data)
output_attacked_snap = model(attacked_snap_color_data)
# Get the predicted class from the model for each case
attacked_pred = output_attacked.max(1, keepdim=True)[1] attacked_pred = output_attacked.max(1, keepdim=True)[1]
kuwahara_pred = output_kuwahara.max(1, keepdim=True)[1] kuwahara_pred = output_kuwahara.max(1, keepdim=True)[1]
bilateral_pred = output_bilateral.max(1, keepdim=True)[1] bilateral_pred = output_bilateral.max(1, keepdim=True)[1]
gaussian_blur_pred = output_gaussian_blur.max(1, keepdim=True)[1] gaussian_blur_pred = output_gaussian_blur.max(1, keepdim=True)[1]
noisy_pred = output_noisy.max(1, keepdim=True)[1] random_noise_pred = output_random_noise.max(1, keepdim=True)[1]
attacked_snap_color_pred = output_attacked_snap.max(1, keepdim=True)[1]
# Count up correct classifications for each case
if orig_pred.item() == target.item(): if orig_pred.item() == target.item():
orig_correct += 1 orig_correct += 1
@ -139,25 +145,31 @@ def test(model, device, test_loader, epsilon):
if gaussian_blur_pred.item() == target.item(): if gaussian_blur_pred.item() == target.item():
gaussian_blur_correct += 1 gaussian_blur_correct += 1
if noisy_pred.item() == target.item(): if random_noise_pred.item() == target.item():
noisy_correct += 1 random_noise_correct += 1
if attacked_snap_color_pred.item() == target.item():
attacked_snap_color_correct += 1
# Calculate the overall accuracy of each case
orig_acc = orig_correct/float(len(test_loader)) orig_acc = orig_correct/float(len(test_loader))
attacked_acc = attacked_correct/float(len(test_loader)) attacked_acc = attacked_correct/float(len(test_loader))
kuwahara_acc = kuwahara_correct/float(len(test_loader)) kuwahara_acc = kuwahara_correct/float(len(test_loader))
bilateral_acc = bilateral_correct/float(len(test_loader)) bilateral_acc = bilateral_correct/float(len(test_loader))
gaussian_blur_acc = gaussian_blur_correct/float(len(test_loader)) gaussian_blur_acc = gaussian_blur_correct/float(len(test_loader))
noisy_acc = noisy_correct/float(len(test_loader)) random_noise_acc = random_noise_correct/float(len(test_loader))
attacked_snap_color_acc = attacked_snap_color_correct/float(len(test_loader))
print(f"Epsilon: {epsilon}") print(f"Epsilon: {epsilon}")
print(f"Original Accuracy = {orig_correct} / {len(test_loader)} = {orig_acc}") print(f"Clean (No Filter) Accuracy = {orig_correct} / {len(test_loader)} = {orig_acc}")
print(f"Attacked Accuracy = {attacked_correct} / {len(test_loader)} = {attacked_acc}") print(f"Attacked (No Filter) Accuracy = {attacked_correct} / {len(test_loader)} = {attacked_acc}")
print(f"Kuwahara Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}") print(f"Attacked (Kuwahara Filter) Accuracy = {kuwahara_correct} / {len(test_loader)} = {kuwahara_acc}")
print(f"Bilateral Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}") print(f"Attacked (Bilateral Filter) Accuracy = {bilateral_correct} / {len(test_loader)} = {bilateral_acc}")
print(f"Gaussian Blur Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}") print(f"Attacked (Gaussian Blur) Accuracy = {gaussian_blur_correct} / {len(test_loader)} = {gaussian_blur_acc}")
print(f"Noisy Accuracy = {noisy_correct} / {len(test_loader)} = {noisy_acc}") print(f"Attacked (Random Noise) Accuracy = {random_noise_correct} / {len(test_loader)} = {random_noise_acc}")
print(f"Attacked (Snapped Color) Accuracy = {attacked_snap_color_correct} / {len(test_loader)} = {attacked_snap_color_acc}")
return attacked_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, noisy_acc return attacked_acc, kuwahara_acc, bilateral_acc, gaussian_blur_acc, random_noise_acc, attacked_snap_color_acc
def filtered(data, batch_size=64, filter="kuwahara"): def filtered(data, batch_size=64, filter="kuwahara"):
@ -168,6 +180,7 @@ def filtered(data, batch_size=64, filter="kuwahara"):
except RuntimeError: except RuntimeError:
images = data.detach().numpy().transpose(0,2,3,1) images = data.detach().numpy().transpose(0,2,3,1)
# Apply the Kuwahara filter # Apply the Kuwahara filter
filtered_images = np.ndarray((batch_size,28,28,1)) filtered_images = np.ndarray((batch_size,28,28,1))
@ -197,6 +210,9 @@ def filtered(data, batch_size=64, filter="kuwahara"):
elif filter == "bilateral": elif filter == "bilateral":
for i in range(batch_size): for i in range(batch_size):
filtered_images[i] = cv2.bilateralFilter(images[i], 5, 50, 50).reshape(filtered_images[i].shape) filtered_images[i] = cv2.bilateralFilter(images[i], 5, 50, 50).reshape(filtered_images[i].shape)
elif filter == "snap_color":
for i in range(batch_size):
filtered_images[i] = (images[i]*4).astype(int).astype(float) / 4
# Modify the data with the filtered image # Modify the data with the filtered image
filtered_images = filtered_images.transpose(0,3,1,2) filtered_images = filtered_images.transpose(0,3,1,2)
@ -206,21 +222,27 @@ attacked_accuracies = []
kuwahara_accuracies = [] kuwahara_accuracies = []
bilateral_accuracies = [] bilateral_accuracies = []
gaussian_blur_accuracies = [] gaussian_blur_accuracies = []
noisy_accuracies = [] random_noise_accuracies = []
attacked_snap_color_accuracies = []
print(f"Model: {pretrained_model}") print(f"Model: {pretrained_model}")
for eps in epsilons: for eps in epsilons:
aacc, kacc, bacc, gacc, nacc = test(model, device, test_loader, eps) aacc, kacc, bacc, gacc, nacc, sacc = test(model, device, test_loader, eps)
attacked_accuracies.append(aacc) attacked_accuracies.append(aacc)
kuwahara_accuracies.append(kacc) kuwahara_accuracies.append(kacc)
bilateral_accuracies.append(bacc) bilateral_accuracies.append(bacc)
gaussian_blur_accuracies.append(gacc) gaussian_blur_accuracies.append(gacc)
noisy_accuracies.append(nacc) random_noise_accuracies.append(nacc)
attacked_snap_color_accuracies.append(sacc)
# Plot the results # Plot the results
plt.plot(epsilons, attacked_accuracies, label="Attacked Accuracy") plt.plot(epsilons, attacked_accuracies, label="Attacked Accuracy")
plt.plot(epsilons, kuwahara_accuracies, label="Kuwahara Accuracy") plt.plot(epsilons, kuwahara_accuracies, label="Kuwahara Accuracy")
plt.plot(epsilons, bilateral_accuracies, label="Bilateral Accuracy")
plt.plot(epsilons, gaussian_blur_accuracies, label="Gaussian Blur Accuracy")
plt.plot(epsilons, random_noise_accuracies, label="Random Noise Accuracy")
plt.plot(epsilons, attacked_snap_color_accuracies, label="Snapped Color Accuracy")
plt.legend() plt.legend()
plt.show() plt.show()

97
Filter_Analysis/wiki/Results.md
View File

@ -46,55 +46,64 @@
## Models Defended with Various Filters ## Models Defended with Various Filters
### Tabulated Results ### Tabulated Results
| $\epsilon$ | FGSM | Kuwahara | Bilateral | Gaussian Blur | Random Noise | | $\epsilon$ | FGSM | Kuwahara | Bilateral | Gaussian Blur | Random Noise | Snapped Color |
|------------|--------|----------|-----------|---------------|--------------| |------------|--------|----------|-----------|---------------|--------------|---------------|
| 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | | 0.05 | 0.9600 | 0.8700 | 0.8902 | 0.9271 | 0.9603 | 0.9781 |
| 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | | 0.10 | 0.8753 | 0.8123 | 0.8133 | 0.8516 | 0.8677 | 0.8818 |
| 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | | 0.15 | 0.7229 | 0.7328 | 0.7098 | 0.7415 | 0.7153 | 0.8408 |
| 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | | 0.20 | 0.5008 | 0.6301 | 0.5683 | 0.5983 | 0.4941 | 0.7496 |
| 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | | 0.25 | 0.2922 | 0.5197 | 0.4381 | 0.4591 | 0.2843 | 0.4301 |
| 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | | 0.30 | 0.1599 | 0.3981 | 0.3364 | 0.3481 | 0.1584 | 0.2091 |
### Plotted Results
![]()
### Raw Program Output ### Raw Program Output
Epsilon: 0.05 Epsilon: 0.05
Original Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked Accuracy = 9600 / 10000 = 0.96 Attacked (No Filter) Accuracy = 9600 / 10000 = 0.96
Kuwahara Accuracy = 8700 / 10000 = 0.87 Attacked (Kuwahara Filter) Accuracy = 8700 / 10000 = 0.87
Bilateral Accuracy = 8902 / 10000 = 0.8902 Attacked (Bilateral Filter) Accuracy = 8902 / 10000 = 0.8902
Gaussian Blur Accuracy = 9271 / 10000 = 0.9271 Attacked (Gaussian Blur) Accuracy = 9271 / 10000 = 0.9271
Noisy Accuracy = 9603 / 10000 = 0.9603 Attacked (Random Noise) Accuracy = 9603 / 10000 = 0.9603
Attacked (Snapped Color) Accuracy = 9781 / 10000 = 0.9781
Epsilon: 0.1 Epsilon: 0.1
Original Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked Accuracy = 8753 / 10000 = 0.8753 Attacked (No Filter) Accuracy = 8753 / 10000 = 0.8753
Kuwahara Accuracy = 8123 / 10000 = 0.8123 Attacked (Kuwahara Filter) Accuracy = 8123 / 10000 = 0.8123
Bilateral Accuracy = 8133 / 10000 = 0.8133 Attacked (Bilateral Filter) Accuracy = 8133 / 10000 = 0.8133
Gaussian Blur Accuracy = 8516 / 10000 = 0.8516 Attacked (Gaussian Blur) Accuracy = 8516 / 10000 = 0.8516
Noisy Accuracy = 8677 / 10000 = 0.8677 Attacked (Random Noise) Accuracy = 8677 / 10000 = 0.8677
Attacked (Snapped Color) Accuracy = 8818 / 10000 = 0.8818
Epsilon: 0.15000000000000002 Epsilon: 0.15000000000000002
Original Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked Accuracy = 7229 / 10000 = 0.7229 Attacked (No Filter) Accuracy = 7229 / 10000 = 0.7229
Kuwahara Accuracy = 7328 / 10000 = 0.7328 Attacked (Kuwahara Filter) Accuracy = 7328 / 10000 = 0.7328
Bilateral Accuracy = 7098 / 10000 = 0.7098 Attacked (Bilateral Filter) Accuracy = 7098 / 10000 = 0.7098
Gaussian Blur Accuracy = 7415 / 10000 = 0.7415 Attacked (Gaussian Blur) Accuracy = 7415 / 10000 = 0.7415
Noisy Accuracy = 7153 / 10000 = 0.7153 Attacked (Random Noise) Accuracy = 7153 / 10000 = 0.7153
Attacked (Snapped Color) Accuracy = 8408 / 10000 = 0.8408
Epsilon: 0.2 Epsilon: 0.2
Original Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked Accuracy = 5008 / 10000 = 0.5008 Attacked (No Filter) Accuracy = 5008 / 10000 = 0.5008
Kuwahara Accuracy = 6301 / 10000 = 0.6301 Attacked (Kuwahara Filter) Accuracy = 6301 / 10000 = 0.6301
Bilateral Accuracy = 5683 / 10000 = 0.5683 Attacked (Bilateral Filter) Accuracy = 5683 / 10000 = 0.5683
Gaussian Blur Accuracy = 5983 / 10000 = 0.5983 Attacked (Gaussian Blur) Accuracy = 5983 / 10000 = 0.5983
Noisy Accuracy = 4941 / 10000 = 0.4941 Attacked (Random Noise) Accuracy = 4941 / 10000 = 0.4941
Attacked (Snapped Color) Accuracy = 7496 / 10000 = 0.7496
Epsilon: 0.25 Epsilon: 0.25
Original Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked Accuracy = 2922 / 10000 = 0.2922 Attacked (No Filter) Accuracy = 2922 / 10000 = 0.2922
Kuwahara Accuracy = 5197 / 10000 = 0.5197 Attacked (Kuwahara Filter) Accuracy = 5197 / 10000 = 0.5197
Bilateral Accuracy = 4381 / 10000 = 0.4381 Attacked (Bilateral Filter) Accuracy = 4381 / 10000 = 0.4381
Gaussian Blur Accuracy = 4591 / 10000 = 0.4591 Attacked (Gaussian Blur) Accuracy = 4591 / 10000 = 0.4591
Noisy Accuracy = 2843 / 10000 = 0.2843 Attacked (Random Noise) Accuracy = 2843 / 10000 = 0.2843
Attacked (Snapped Color) Accuracy = 4301 / 10000 = 0.4301
Epsilon: 0.3 Epsilon: 0.3
Original Accuracy = 9920 / 10000 = 0.992 Clean (No Filter) Accuracy = 9920 / 10000 = 0.992
Attacked Accuracy = 1599 / 10000 = 0.1599 Attacked (No Filter) Accuracy = 1599 / 10000 = 0.1599
Kuwahara Accuracy = 3981 / 10000 = 0.3981 Attacked (Kuwahara Filter) Accuracy = 3981 / 10000 = 0.3981
Bilateral Accuracy = 3364 / 10000 = 0.3364 Attacked (Bilateral Filter) Accuracy = 3364 / 10000 = 0.3364
Gaussian Blur Accuracy = 3481 / 10000 = 0.3481 Attacked (Gaussian Blur) Accuracy = 3481 / 10000 = 0.3481
Noisy Accuracy = 1584 / 10000 = 0.1584 Attacked (Random Noise) Accuracy = 1584 / 10000 = 0.1584
Attacked (Snapped Color) Accuracy = 2091 / 10000 = 0.2091