How to use the cleverhans.utils_tf.model_eval function in cleverhans

To help you get started, we’ve selected a few cleverhans examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github tensorflow / cleverhans / examples / iterative_advtrain / train.py View on Github external
def evaluate_2():
        # Accuracy of adversarially trained model on legitimate test inputs
        eval_params = {'batch_size': batch_size}
        accuracy = model_eval(sess, x, y, preds_2, X_test, Y_test,
                              args=eval_params)
        print('Test accuracy on legitimate examples: %0.4f' % accuracy)
        report.adv_train_clean_eval = accuracy

        # Accuracy of the adversarially trained model on adversarial examples
        accuracy = model_eval(sess, x, y, preds_2_adv, X_test,
                              Y_test, args=eval_params)
        print('Test accuracy on adversarial examples: %0.4f' % accuracy)
        report.adv_train_adv_eval = accuracy
github tensorflow / cleverhans / cleverhans_tutorials / mnist_tutorial_keras_tf.py View on Github external
report.adv_train_clean_eval = accuracy

    # Accuracy of the adversarially trained model on adversarial examples
    accuracy = model_eval(sess, x, y, preds_2_adv, x_test,
                          y_test, args=eval_params)
    print('Test accuracy on adversarial examples: %0.4f' % accuracy)
    report.adv_train_adv_eval = accuracy

  # Perform and evaluate adversarial training
  train(sess, loss_2, x_train, y_train, evaluate=evaluate_2,
        args=train_params, rng=rng)

  # Calculate training errors
  if testing:
    eval_params = {'batch_size': batch_size}
    accuracy = model_eval(sess, x, y, preds_2, x_train, y_train,
                          args=eval_params)
    report.train_adv_train_clean_eval = accuracy
    accuracy = model_eval(sess, x, y, preds_2_adv, x_train,
                          y_train, args=eval_params)
    report.train_adv_train_adv_eval = accuracy

  return report
github sakishinoda / tf-ssl / vat_ladder / adversarial.py View on Github external
import numpy as np
                ord = np.inf
            elif p.ord == '1':
                ord = 1
            else:
                ord = 2

            fgsm_params = {'eps': 0.3, 'ord': ord}
            fgsm = FastGradientMethod(model, sess=sess)

            adv_x = fgsm.generate(x, **fgsm_params)
            adv_x = tf.stop_gradient(adv_x)
            preds_adv = model.get_probs(adv_x)

            # Evaluate the accuracy of the MNIST model on adversarial examples
            acc = model_eval(sess, x, y, preds_adv, X_test, Y_test,
                             feed={model.g['train_flag']: False},
                             args=eval_par)
            aer = 100 * (1 - acc)
            results['adv_aer'] = aer
            # print('Test AER on adversarial examples: {:0.4f} %'.format(aer))
            print(results['checkpoint'], results['normal_aer'], results[
                'adv_aer'], sep=',')

    return results
github IBM / ZOO-Attack / mnist_blackbox.py View on Github external
# Train an MNIST model
    if FLAGS.load_pretrain:
        tf_model_load(sess)
    else:
        train_params = {
            'nb_epochs': nb_epochs,
            'batch_size': batch_size,
            'learning_rate': learning_rate
        }
        model_train(sess, x, y, predictions, X_train, Y_train, verbose=False, save=True,
                    args=train_params)

    # Print out the accuracy on legitimate data
    eval_params = {'batch_size': batch_size}
    accuracy = model_eval(sess, x, y, predictions, X_test, Y_test,
                          args=eval_params)
    print('Test accuracy of black-box on legitimate test '
          'examples: ' + str(accuracy))

    return model, predictions, accuracy
github IBM / ZOO-Attack / mnist_blackbox.py View on Github external
attacker_params['y_target'] = np.array(adv_ys, dtype=np.float32)
            # duplicate the inputs by 9 times
            adv_inputs = np.array([[instance] * 9 for instance in X_test],
                                  dtype=np.float32)
            adv_inputs = adv_inputs.reshape((X_test.shape[0] * 9, 28, 28, 1))
            # also update the mask
            mask = np.repeat(mask, 9)
            ori_labels = np.repeat(Y_test, 9, axis=0)
        else:
            adv_inputs = X_test
            ori_labels = Y_test
        attacker = cwl2

    if attack == "fgsm":
        # Evaluate the accuracy of the "black-box" model on adversarial examples
        accuracy = model_eval(sess, x, y, model(x_adv_sub), adv_inputs, ori_labels,
                              args=eval_params)
        print('Test accuracy of oracle on adversarial examples generated '
              'using the substitute: ' + str(accuracy))
        accuracies['bbox_on_sub_adv_ex'] = accuracy

    time_start = time.time()
    # Evaluate the accuracy of the "black-box" model on adversarial examples
    x_adv_sub_np = attacker.generate_np(adv_inputs, **attacker_params)
    accuracy = model_eval(sess, x, y, bbox_preds, x_adv_sub_np, ori_labels,
                          args=eval_params)
    print('Test accuracy of oracle on adversarial examples generated '
          'using the substitute (NP): ' + str(accuracy))
    accuracies['bbox_on_sub_adv_ex'] = accuracy
    time_end = time.time()
    print('Attack time:', time_end - time_start)
github kabkabm / defensegan / whitebox.py View on Github external
if FLAGS.defense_type == 'defense_gan':
        preds_adv = model.get_probs(adv_x)

        num_dims = len(images_pl.get_shape())
        avg_inds = list(range(1, num_dims))
        diff_op = tf.reduce_mean(tf.square(adv_x - images_pl), axis=avg_inds)
        acc_adv, roc_info = model_eval_gan(
            sess, images_pl, labels_pl, preds_adv, None,
            test_images=test_images, test_labels=test_labels, args=eval_par,
            feed={K.learning_phase(): 0}, diff_op=diff_op,
        )
        print('Test accuracy on adversarial examples: %0.4f\n' % acc_adv)
        return acc_adv, 0, roc_info
    else:
        preds_adv = model(adv_x)
        acc_adv = model_eval(sess, images_pl, labels_pl, preds_adv, test_images, test_labels,
                             args=eval_par,
                             feed={K.learning_phase(): 0})
        print('Test accuracy on adversarial examples: %0.4f\n' % acc_adv)

        return acc_adv, 0, None
github IBM / ZOO-Attack / mnist_blackbox.py View on Github external
adv_inputs = X_test
            ori_labels = Y_test
        attacker = cwl2

    if attack == "fgsm":
        # Evaluate the accuracy of the "black-box" model on adversarial examples
        accuracy = model_eval(sess, x, y, model(x_adv_sub), adv_inputs, ori_labels,
                              args=eval_params)
        print('Test accuracy of oracle on adversarial examples generated '
              'using the substitute: ' + str(accuracy))
        accuracies['bbox_on_sub_adv_ex'] = accuracy

    time_start = time.time()
    # Evaluate the accuracy of the "black-box" model on adversarial examples
    x_adv_sub_np = attacker.generate_np(adv_inputs, **attacker_params)
    accuracy = model_eval(sess, x, y, bbox_preds, x_adv_sub_np, ori_labels,
                          args=eval_params)
    print('Test accuracy of oracle on adversarial examples generated '
          'using the substitute (NP): ' + str(accuracy))
    accuracies['bbox_on_sub_adv_ex'] = accuracy
    time_end = time.time()
    print('Attack time:', time_end - time_start)

    # Evaluate the targeted attack
    bbox_adv_predict = batch_eval(sess, [x], [bbox_preds], [x_adv_sub_np],
                          args=eval_params)[0]
    bbox_adv_class = np.argmax(bbox_adv_predict, axis = 1)
    true_class = np.argmax(ori_labels, axis = 1)
    untargeted_success = np.mean(bbox_adv_class != true_class)
    print('Untargeted attack success rate:', untargeted_success)
    accuracies['untargeted_success'] = untargeted_success
    if targeted:
github tensorflow / cleverhans / cleverhans_tutorials / mnist_tutorial_keras_tf.py View on Github external
accuracy = model_eval(sess, x, y, preds_2_adv, x_test,
                          y_test, args=eval_params)
    print('Test accuracy on adversarial examples: %0.4f' % accuracy)
    report.adv_train_adv_eval = accuracy

  # Perform and evaluate adversarial training
  train(sess, loss_2, x_train, y_train, evaluate=evaluate_2,
        args=train_params, rng=rng)

  # Calculate training errors
  if testing:
    eval_params = {'batch_size': batch_size}
    accuracy = model_eval(sess, x, y, preds_2, x_train, y_train,
                          args=eval_params)
    report.train_adv_train_clean_eval = accuracy
    accuracy = model_eval(sess, x, y, preds_2_adv, x_train,
                          y_train, args=eval_params)
    report.train_adv_train_adv_eval = accuracy

  return report
github tensorflow / cleverhans / examples / linear_extrapolation_plot_example.py View on Github external
# Plot the linear extrapolation plot for clean model
    log_prob_adv_array = get_logits_over_interval(
        sess, wrap, X_slice, fgsm_params)
    linear_extrapolation_plot(log_prob_adv_array, Y_slice,
                              'lep_clean.png')

    # Plot the linear extrapolation plot for adv model
    log_prob_adv_array = get_logits_over_interval(
        sess, wrap_2, X_slice, fgsm_params)
    linear_extrapolation_plot(log_prob_adv_array, Y_slice,
                              'lep_adv.png')

    # Calculate training errors
    if testing:
        eval_params = {'batch_size': batch_size}
        accuracy = model_eval(sess, x, y, preds_2, X_train, Y_train,
                              args=eval_params)
        report.train_adv_train_clean_eval = accuracy
        accuracy = model_eval(sess, x, y, preds_2_adv, X_train,
                              Y_train, args=eval_params)
        report.train_adv_train_adv_eval = accuracy

    return report
github tensorflow / cleverhans / cleverhans_tutorials / mnist_tutorial_cw.py View on Github external
yname: adv_ys,
               'max_iterations': attack_iterations,
               'learning_rate': CW_LEARNING_RATE,
               'batch_size': cw_params_batch_size,
               'initial_const': 10}

  adv = cw.generate_np(adv_inputs,
                       **cw_params)

  eval_params = {'batch_size': np.minimum(nb_classes, source_samples)}
  if targeted:
    adv_accuracy = model_eval(
        sess, x, y, preds, adv, adv_ys, args=eval_params)
  else:
    if viz_enabled:
      err = model_eval(sess, x, y, preds, adv, y_test[idxs], args=eval_params)
      adv_accuracy = 1 - err
    else:
      err = model_eval(sess, x, y, preds, adv, y_test[:source_samples],
                       args=eval_params)
      adv_accuracy = 1 - err

  if viz_enabled:
    for j in range(nb_classes):
      if targeted:
        for i in range(nb_classes):
          grid_viz_data[i, j] = adv[i * nb_classes + j]
      else:
        grid_viz_data[j, 0] = adv_inputs[j]
        grid_viz_data[j, 1] = adv[j]

    print(grid_viz_data.shape)