How to use the openai.train.find_trainable_variables function in openai

To help you get started, we’ve selected a few openai examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github Separius / BERT-keras / tests / test_transformer.py View on Github external
n_transfer = 1 + 12 * 12

        def model(X, train=False, reuse=False):
            with tf.variable_scope('model', reuse=reuse):
                we = tf.get_variable("we", [n_vocab + TextEncoder.SPECIAL_COUNT + n_ctx, n_embd],
                                     initializer=tf.random_normal_initializer(stddev=0.02))
                we = dropout(we, embd_pdrop, train)
                h = embed(X, we)
                for layer in range(n_layer):
                    h = block(h, 'h%d' % layer, train=train, scale=True)
                return h

        X_train = tf.placeholder(tf.int32, [n_batch_train, n_ctx, 2])
        res = model(X_train)

        params = find_trainable_variables('model')
        sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
        sess.run(tf.global_variables_initializer())

        with open('openai/model/params_shapes.json') as f:
            shapes = json.load(f)
        offsets = np.cumsum([np.prod(shape) for shape in shapes])
        init_params = [np.load('openai/model/params_{}.npy'.format(n)) for n in range(10)]
        init_params = np.split(np.concatenate(init_params, 0), offsets)[:-1]
        init_params = [param.reshape(shape) for param, shape in zip(init_params, shapes)]
        init_params[0] = init_params[0][:n_ctx]
        init_params[0] = np.concatenate(
            [init_params[1], (np.random.randn(TextEncoder.SPECIAL_COUNT, n_embd) * 0.02).astype(np.float32),
             init_params[0]], 0)
        del init_params[1]

        sess.run([p.assign(ip) for p, ip in zip(params[:n_transfer], init_params[:n_transfer])])