将现有项目转换为 TensorFlow

Question

我有一个基于以下代码的现有项目：https ://gist.github.com/karpathy/d4dee566867f8291f086 。作为学习 TensorFlow 的一个练习，我试图将此代码转换为使用 TF，但是，许多 API 文档似乎都期望我要移植的代码中不存在各种术语的先验知识。我假设我会使用 SimpleRNN 或 RNN 的某个子类。这段代码的基本 TensorFlow 版本会是什么样子？

Answer 1

在获得了一些空闲时间和大量的摆弄之后，我想出了类似的东西：

import numpy as np

import tensorflow as tf

from tensorflow.keras import layers, initializers

from tensorflow.keras.optimizers import Adagrad

SCALE = 100.0

def makeXVector(ix):

    x = np.zeros((vocab_size, ))

    x[ix] = 1.0

    return x

with open('input.txt', 'r') as f:

    print('Reading and tokenising input.txt')

    data = f.read()

print('Collecting chars')

chars = list(set(data))

data_size, vocab_size = len(data), len(chars)

char_to_ix = { tok:i for i,tok in enumerate(chars) }

ix_to_char = { i:tok for i,tok in enumerate(chars) }

# hyperparameters

hidden_size = 300 # size of hidden layer of neurons

seq_length = 25 # number of steps to unroll the RNN for

learning_rate = 1e-1

model = tf.keras.Sequential()

model.add(

    layers.SimpleRNN(

      hidden_size,

      activation='tanh',

      use_bias=True,

      stateful=True,

      bias_initializer=initializers.Zeros(),

      # (batch_size, timesteps, input_dim)

      batch_input_shape=np.array((1, 1, vocab_size))

    )

)

model.add(

    layers.Dense(

      vocab_size,

      activation='softmax',

      use_bias=True,

      bias_initializer=initializers.Zeros(),

    )

)

model.compile(

    optimizer=Adagrad(learning_rate=learning_rate),

    loss='categorical_crossentropy'

)

n, p = 0, 0

smooth_loss = -np.log(1.0/vocab_size)*seq_length # loss at iteration 0

def sample(model, seed_ix, n):

    x = makeXVector(seed_ix)

    ixes = []

    for t in range(n):

      p = np.array(model.predict(np.array([[x.ravel()]]))).ravel()

      # Select a random character based on the probability

      ix = np.random.choice(list(range(vocab_size)), p=p)

      x = makeXVector(ix)

      ixes.append(ix)

    return ixes

print('data has {:d} characters, {:d} unique.'.format(data_size, vocab_size))

try:

    while True:

      # prepare inputs (we're sweeping from left to right in steps seq_length long)

      if p+seq_length+1 >= len(data) or n == 0: 

        p = 0 # go from start of data

      inputs = [char_to_ix[tok] for tok in data[p:p+seq_length]]

      targets = [char_to_ix[tok] for tok in data[p+1:p+seq_length+1]]

      # sample from the model now and then

      if n % 100 == 0:

        sample_ix = sample(model, inputs[0], 200)

        txt = ' '.join((ix_to_char[ix][0] for ix in sample_ix))

        print('----\n{}\n----'.format(txt))

      hist = model.fit(

        np.array([[inp] for inp in map(makeXVector, inputs)]),

        np.array([targ for targ in map(makeXVector, targets)]),

        verbose=0

      )

      loss = hist.history['loss'][-1]

      smooth_loss = smooth_loss * 0.999 + loss * 0.001

      if n % 100 == 0: print('iter {:d}, loss: {:f}'.format(n, smooth_loss)) # print progress

      p += seq_length # move data pointer

      n += 1 # iteration counter 

except KeyboardInterrupt:

    pass