详解PLANET代码(tensorflow)如何加入SAC功能

train_loss = tf.cond(
    tf.equal(phase, 'train'),
    lambda: loss,
    lambda: 0 * tf.get_variable('dummy_loss', (), tf.float32))

def random_episodes(env_ctor, num_episodes, output_dir=None):

with tf.variable_scope('simulation'):
  sim_returns = []
  for name, params in config.sim_summaries.items():
    # These are expensive and equivalent for train and test phases, so only
    # do one of them.
    sim_summary, sim_return = tf.cond(
        tf.equal(graph.phase, 'test'),
        lambda: utility.simulate_episodes(config, params, graph, name),
        lambda: ('', 0.0),
        name='should_simulate_' + params.task.name)
    summaries.append(sim_summary)
    sim_returns.append(sim_return)

def random_episodes(env_ctor, num_episodes, output_dir=None):
  env = env_ctor()  # env is an <ExternalProcess object>.
  env = wrappers.CollectGymDataset(env, output_dir)
  episodes = []
  for _ in range(num_episodes):
    policy = lambda env, obs: env.action_space.sample()
    done = False
    obs = env.reset()
    # cnt = 0
    while not done:
      action = policy(env, obs)
      obs, _, done, info = env.step(action)  # env.step
    #   cnt += 1
    # print(cnt)
    episodes.append(info['episode'])  # if done is True, info stores the 'episode' information and 'episode' is written in a file(e.g. "~/planet/log_debug/00001/test_episodes").
  return episodes

def _process_step(self, action, observ, reward, done, info):

def _read_episodes_scan(
    directory, batch_size, every, max_episodes=None, **kwargs):
  recent = {}
  cache = {}
  while True:
    index = 0
    for episode in np.random.permutation(list(recent.values())):
      yield episode
      index += 1
      if index > every / 2:
        break
    for episode in np.random.permutation(list(cache.values())):
      index += 1
      yield episode
      if index > every:
        break
    # At the end of the epoch, add new episodes to the cache.
    # print(index, len(recent), max_episodes, len(cache))
    cache.update(recent)
    recent = {}
    filenames = tf.gfile.Glob(os.path.join(directory, '*.npz'))
    filenames = [filename for filename in filenames if filename not in cache]
    if max_episodes:
      filenames = filenames[:max_episodes - len(cache)]
    for filename in filenames:
      recent[filename] = _read_episode(filename, **kwargs)

train = tf.data.Dataset.from_generator(                                         # train.output_shapes: e.g. 'image' shape(?, 64, 64, 3)
    functools.partial(loader, train_dir, shape[0], **kwargs), dtypes, shapes)

data = get_batch(datasets, trainer.phase, trainer.reset)

def random_episodes(env_ctor, num_episodes, output_dir=None):
  env = env_ctor()  # env is an <ExternalProcess object>.
  env = wrappers.CollectGymDataset(env, output_dir)
  episodes = []
  for _ in range(num_episodes):
    policy = lambda env, obs: env.action_space.sample()
    done = False
    obs = env.reset()
    # cnt = 0
    while not done:
      action = policy(env, obs)
      obs, _, done, info = env.step(action)  # env.step
    #   cnt += 1
    # print(cnt)
    episodes.append(info['episode'])  # if done is True, info stores the 'episode' information and 'episode' is written in a file(e.g. "~/planet/log_debug/00001/test_episodes").
  return episodes

def compute_losses(
    loss_scales, cell, heads, step, target, prior, posterior, mask,
    free_nats=None, debug=False):
  features = cell.features_from_state(posterior)      # [s,h]
  losses = {}
  for key, scale in loss_scales.items():
    # Skip losses with zero or None scale to save computation.
    if not scale:
      continue
    elif key == 'divergence':
      loss = cell.divergence_from_states(posterior, prior, mask)
      if free_nats is not None:
        loss = tf.maximum(tf.cast(free_nats, tf.float32), loss)
      loss = tf.reduce_sum(loss, 1) / tf.reduce_sum(tf.to_float(mask), 1)
    elif key == 'global_divergence':
      global_prior = {
          'mean': tf.zeros_like(prior['mean']),
          'stddev': tf.ones_like(prior['stddev'])}
      loss = cell.divergence_from_states(posterior, global_prior, mask)
      loss = tf.reduce_sum(loss, 1) / tf.reduce_sum(tf.to_float(mask), 1)
    elif key in heads:
      output = heads[key](features)   # decoder is used.
      loss = -tools.mask(output.log_prob(target[key]), mask)
    else:
      message = "Loss scale of head '{}' is not used."
      print(message.format(key))
      continue
    # Average over the batch and normalize by the maximum chunk length.
    loss = tf.reduce_mean(loss)
    losses[key] = tf.check_numerics(loss, key) if debug else loss
  return losses

def add_saver(
    self, include=r'.*', exclude=r'.^', logdir=None, load=True, save=True,
    checkpoint=None):
  """Add a saver to save or load variables.

  Args:
    include: One or more regexes to match variable names to include.
    exclude: One or more regexes to match variable names to exclude.
    logdir: Directory for saver to store and search for checkpoints.
    load: Whether to use the saver to restore variables.
    save: Whether to use the saver to save variables.
    checkpoint: Checkpoint name to load; None for newest.
  """
  variables = tools.filter_variables(include, exclude)
  saver = tf.train.Saver(variables, keep_checkpoint_every_n_hours=2)
  if load: