Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_fit_predict(self):
(x_train, y_train), (x_test, y_test) = self.mnist
attack = FastGradientMethod(self.classifier_k)
x_test_adv = attack.generate(x_test)
preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier_k, attack)
adv_trainer.fit(x_train, y_train, nb_epochs=5, batch_size=128)
preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST
self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP)
logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100))
logger.info('Accuracy after adversarial training: %.2f%%', (acc_new * 100))
def test_two_attacks(self):
(x_train, y_train), (x_test, y_test) = self.mnist
attack1 = FastGradientMethod(self.classifier_k)
attack2 = DeepFool(self.classifier_tf)
x_test_adv = attack1.generate(x_test)
preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier_k, attacks=[attack1, attack2])
adv_trainer.fit(x_train, y_train, nb_epochs=5, batch_size=128)
preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST
# No reason to assert the newer accuracy is higher. It might go down slightly
self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP)
logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100))
logger.info('\nAccuracy after adversarial training: %.2f%%', (acc_new * 100))
def test_targeted_attack_error(self):
"""
Test the adversarial trainer using a targeted attack, which will currently result in a
NotImplementError.
:return: None
"""
(x_train, y_train), (_, _) = self.mnist
params = {'nb_epochs': 2, 'batch_size': BATCH_SIZE}
classifier = self.classifier_k
adv = FastGradientMethod(classifier, targeted=True)
adv_trainer = AdversarialTrainer(classifier, attacks=adv)
self.assertRaises(NotImplementedError, adv_trainer.fit, x_train, y_train, **params)
def test_iris_tf(self):
classifier, _ = get_iris_classifier_tf()
# Test untargeted attack
attack = FastGradientMethod(classifier, eps=.1)
x_test_adv = attack.generate(self.x_test)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test, axis=1) == predictions_adv).all())
accuracy = np.sum(predictions_adv == np.argmax(self.y_test, axis=1)) / self.y_test.shape[0]
logger.info('Accuracy on Iris with FGM adversarial examples: %.2f%%', (accuracy * 100))
# Test targeted attack
targets = random_targets(self.y_test, nb_classes=3)
attack = FastGradientMethod(classifier, targeted=True, eps=.1, batch_size=128)
x_test_adv = attack.generate(self.x_test, **{'y': targets})
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
def test_iris_k_unbounded(self):
classifier, _ = get_iris_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model, use_logits=False, channel_index=1)
attack = FastGradientMethod(classifier, eps=1)
x_test_adv = attack.generate(self.x_test)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertTrue((x_test_adv > 1).any())
self.assertTrue((x_test_adv < 0).any())
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test, axis=1) == predictions_adv).all())
accuracy = np.sum(predictions_adv == np.argmax(self.y_test, axis=1)) / self.y_test.shape[0]
logger.info('Accuracy on Iris with FGM adversarial examples: %.2f%%', (accuracy * 100))
def test_classifier_type_check_fail_gradients(self):
# Use a test classifier not providing gradients required by white-box attack
from art.classifiers.scikitlearn import ScikitlearnDecisionTreeClassifier
from sklearn.tree import DecisionTreeClassifier
classifier = ScikitlearnDecisionTreeClassifier(model=DecisionTreeClassifier())
with self.assertRaises(TypeError) as context:
_ = FastGradientMethod(classifier=classifier)
self.assertIn('For `FastGradientMethod` classifier must be an instance of '
'`art.classifiers.classifier.ClassifierGradients`, the provided classifier is instance of '
'(,).', str(context.exception))
def test_binary_activation_detector(self):
"""
Test the binary activation detector end-to-end.
:return:
"""
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# Keras classifier
classifier = get_classifier_kr()
# Generate adversarial samples:
attacker = FastGradientMethod(classifier, eps=0.1)
x_train_adv = attacker.generate(x_train[:NB_TRAIN])
x_test_adv = attacker.generate(x_test[:NB_TRAIN])
# Compile training data for detector:
x_train_detector = np.concatenate((x_train[:NB_TRAIN], x_train_adv), axis=0)
y_train_detector = np.concatenate((np.array([[1, 0]] * NB_TRAIN), np.array([[0, 1]] * NB_TRAIN)), axis=0)
# Create a simple CNN for the detector
activation_shape = classifier.get_activations(x_test[:1], 0, batch_size=128).shape[1:]
number_outputs = 2
model = Sequential()
model.add(MaxPooling2D(pool_size=(2, 2), input_shape=activation_shape))
model.add(Flatten())
model.add(Dense(number_outputs, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(lr=0.01),
metrics=['accuracy'])
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
classifier = KerasClassifier(model=model, clip_values=(min_, max_))
classifier.fit(x_train, y_train, nb_epochs=5, batch_size=128)
# Evaluate the classifier on the test set
preds = np.argmax(classifier.predict(x_test), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / y_test.shape[0]
print("\nTest accuracy: %.2f%%" % (acc * 100))
# Craft adversarial samples with FGSM
epsilon = .1 # Maximum perturbation
adv_crafter = FastGradientMethod(classifier, eps=epsilon)
x_test_adv = adv_crafter.generate(x=x_test)
# Evaluate the classifier on the adversarial examples
preds = np.argmax(classifier.predict(x_test_adv), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / y_test.shape[0]
print("\nTest accuracy on adversarial sample: %.2f%%" % (acc * 100))
classifier = TFClassifier(clip_values=(min_pixel_value, max_pixel_value), input_ph=input_ph, output=logits,
labels_ph=labels_ph, train=train, loss=loss, learning=None, sess=sess)
# Step 4: Train the ART classifier
classifier.fit(x_train, y_train, batch_size=64, nb_epochs=3)
# Step 5: Evaluate the ART classifier on benign test examples
predictions = classifier.predict(x_test)
accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)
print('Accuracy on benign test examples: {}%'.format(accuracy * 100))
# Step 6: Generate adversarial test examples
attack = FastGradientMethod(classifier=classifier, eps=0.2)
x_test_adv = attack.generate(x=x_test)
# Step 7: Evaluate the ART classifier on adversarial test examples
predictions = classifier.predict(x_test_adv)
accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)
print('Accuracy on adversarial test examples: {}%'.format(accuracy * 100))
classifier = MXClassifier(model=model, clip_values=(min_pixel_value, max_pixel_value), loss=loss,
input_shape=(28, 28, 1), nb_classes=10, optimizer=trainer, ctx=None, channel_index=1,
defences=None, preprocessing=(0, 1))
# Step 4: Train the ART classifier
classifier.fit(x_train, y_train, batch_size=64, nb_epochs=3)
# Step 5: Evaluate the ART classifier on benign test examples
predictions = classifier.predict(x_test)
accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)
print('Accuracy on benign test examples: {}%'.format(accuracy * 100))
# Step 6: Generate adversarial test examples
attack = FastGradientMethod(classifier=classifier, eps=0.2)
x_test_adv = attack.generate(x=x_test)
# Step 7: Evaluate the ART classifier on adversarial test examples
predictions = classifier.predict(x_test_adv)
accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)
print('Accuracy on adversarial test examples: {}%'.format(accuracy * 100))