exploration.cpp File Reference

#include <portability.h>
#include "exploration.h"
#include <stdlib.h>
#include <math.h>
#include <float.h>
#include <list>
#include <limits>
#include "agent.h"
#include "soar_rand.h"
#include "xml.h"
#include "print.h"
#include "soar_TraceNames.h"
#include "gsysparam.h"
#include "reinforcement_learning.h"
#include "misc.h"
#include "utilities.h"
#include "instantiations.h"

Go to the source code of this file.

Functions
exploration_parameter *	exploration_add_parameter (double value, bool(*val_func)(double), const char *name)
preference *	exploration_boltzmann_select (agent *my_agent, preference *candidates)
preference *	exploration_choose_according_to_policy (agent *my_agent, slot *s, preference *candidates)
void	exploration_compute_value_of_candidate (agent *my_agent, preference *cand, slot *s, double default_value)
const int	exploration_convert_parameter (agent *my_agent, const char *name)
const char *	exploration_convert_parameter (agent *my_agent, const int parameter)
const int	exploration_convert_policy (const char *policy_name)
const char *	exploration_convert_policy (const int policy)
const int	exploration_convert_reduction_policy (const char *policy_name)
const char *	exploration_convert_reduction_policy (const int policy)
preference *	exploration_epsilon_greedy_select (agent *my_agent, preference *candidates)
bool	exploration_get_auto_update (agent *my_agent)
preference *	exploration_get_highest_q_value_pref (preference *candidates)
double	exploration_get_parameter_value (agent *my_agent, const char *parameter)
double	exploration_get_parameter_value (agent *my_agent, const int parameter)
const int	exploration_get_policy (agent *my_agent)
const int	exploration_get_reduction_policy (agent *my_agent, const char *parameter)
const int	exploration_get_reduction_policy (agent *my_agent, const int parameter)
double	exploration_get_reduction_rate (agent *my_agent, const char *parameter, const char *policy_name)
double	exploration_get_reduction_rate (agent *my_agent, const int parameter, const int policy)
preference *	exploration_probabilistically_select (preference *candidates)
double	exploration_probability_according_to_policy (agent *my_agent, slot *s, preference *candidates, preference *selection)
preference *	exploration_randomly_select (preference *candidates)
bool	exploration_set_auto_update (agent *my_agent, bool setting)
bool	exploration_set_parameter_value (agent *my_agent, const char *name, double value)
bool	exploration_set_parameter_value (agent *my_agent, const int parameter, double value)
bool	exploration_set_policy (agent *my_agent, const char *policy_name)
bool	exploration_set_policy (agent *my_agent, const int policy)
bool	exploration_set_reduction_policy (agent *my_agent, const char *parameter, const char *policy_name)
bool	exploration_set_reduction_policy (agent *my_agent, const int parameter, const int policy)
bool	exploration_set_reduction_rate (agent *my_agent, const char *parameter, const char *policy_name, double reduction_rate)
bool	exploration_set_reduction_rate (agent *my_agent, const int parameter, const int policy, double reduction_rate)
void	exploration_update_parameters (agent *my_agent)
bool	exploration_valid_exponential (double reduction_rate)
bool	exploration_valid_linear (double reduction_rate)
const bool	exploration_valid_parameter (agent *my_agent, const char *name)
const bool	exploration_valid_parameter (agent *my_agent, const int parameter)
bool	exploration_valid_parameter_value (agent *my_agent, const char *name, double value)
bool	exploration_valid_parameter_value (agent *my_agent, const int parameter, double value)
bool	exploration_valid_policy (const char *policy_name)
bool	exploration_valid_policy (const int policy)
bool	exploration_valid_reduction_policy (agent *, const char *, const char *policy_name)
bool	exploration_valid_reduction_policy (agent *, const char *, const int policy)
bool	exploration_valid_reduction_policy (agent *, const int, const int policy)
bool	exploration_valid_reduction_rate (agent *my_agent, const char *parameter, const char *policy_name, double reduction_rate)
bool	exploration_valid_reduction_rate (agent *my_agent, const int parameter, const int policy, double reduction_rate)
bool	exploration_validate_epsilon (double value)
bool	exploration_validate_temperature (double value)

Function Documentation

exploration_parameter* exploration_add_parameter	(	double	value,
		bool(*)(double)	val_func,
		const char *	name
	)

Definition at line 126 of file exploration.cpp.

References EXPLORATION_REDUCTION_EXPONENTIAL, EXPLORATION_REDUCTION_LINEAR, exploration_parameter_struct::name, exploration_parameter_struct::rates, exploration_parameter_struct::reduction_policy, exploration_parameter_struct::val_func, and exploration_parameter_struct::value.

Referenced by create_soar_agent().

{
 // new parameter entry
 exploration_parameter * const newbie = new exploration_parameter;
 newbie->value = value;
 newbie->name = name;
 newbie->reduction_policy = EXPLORATION_REDUCTION_EXPONENTIAL;
 newbie->val_func = val_func;
 newbie->rates[ EXPLORATION_REDUCTION_EXPONENTIAL ] = 1;
 newbie->rates[ EXPLORATION_REDUCTION_LINEAR ] = 0;

 return newbie;
}

preference* exploration_boltzmann_select	(	agent *	my_agent,
		preference *	candidates
	)

Definition at line 753 of file exploration.cpp.

References exploration_get_parameter_value(), EXPLORATION_PARAM_TEMPERATURE, preference_struct::next_candidate, NIL, preference_struct::numeric_value, print(), print_with_symbols(), SoarRand(), agent_struct::sysparams, TRACE_INDIFFERENT_SYSPARAM, preference_struct::value, xml_att_val(), xml_begin_tag(), and xml_end_tag().

Referenced by exploration_choose_according_to_policy().

{
 double t = exploration_get_parameter_value( my_agent, EXPLORATION_PARAM_TEMPERATURE );
 double maxq;
 preference* c;
 
 maxq = candidates->numeric_value;
 for (c = candidates->next_candidate; c; c = c->next_candidate) {
  if (maxq < c->numeric_value)
   maxq = c->numeric_value;
 }
 
 double exptotal = 0.0;
 std::list<double> expvals;
 std::list<double>::iterator i;

 for (c = candidates; c; c = c->next_candidate) {
  // equivalent to exp((c->numeric_value / t) - (maxq / t)) but safer against overflow
  double v = exp((c->numeric_value - maxq) / t);
  expvals.push_back(v);
  exptotal += v;
 }
 
 // output trace information
 if ( my_agent->sysparams[ TRACE_INDIFFERENT_SYSPARAM ] )
 {
  for (c = candidates, i = expvals.begin(); c; c = c->next_candidate, i++)
  {
   double prob = *i / exptotal;
   print_with_symbols( my_agent, "\n Candidate %y:  ", c->value );
   print( my_agent, "Value (Sum) = %f, (Prob) = %f", c->numeric_value, prob );
   xml_begin_tag( my_agent, kTagCandidate );
   xml_att_val( my_agent, kCandidateName, c->value );
   xml_att_val( my_agent, kCandidateType, kCandidateTypeSum );
   xml_att_val( my_agent, kCandidateValue, c->numeric_value );
   xml_att_val( my_agent, kCandidateExpValue, prob );
   xml_end_tag( my_agent, kTagCandidate );
  }
 }

 double r = SoarRand(exptotal);
 double sum = 0.0;

 for (c = candidates, i = expvals.begin(); c; c = c->next_candidate, i++) {
  sum += *i;
  if (sum >= r)
   return c;
 }
 
 return NIL;
}

preference* exploration_choose_according_to_policy	(	agent *	my_agent,
		slot *	s,
		preference *	candidates
	)

Definition at line 503 of file exploration.cpp.

References exploration_boltzmann_select(), exploration_compute_value_of_candidate(), exploration_epsilon_greedy_select(), exploration_get_policy(), exploration_probabilistically_select(), exploration_randomly_select(), soar_module::constant_param< T >::get_value(), slot_struct::id, rl_param_container::learning_policy, preference_struct::next_candidate, preference_struct::numeric_value, rl_param_container::q, preference_struct::rl_contribution, rl_enabled(), agent_struct::rl_params, rl_perform_update(), rl_tabulate_reward_values(), rl_watkins_clear(), rl_param_container::sarsa, USER_SELECT_BOLTZMANN, USER_SELECT_E_GREEDY, USER_SELECT_FIRST, USER_SELECT_LAST, USER_SELECT_RANDOM, and USER_SELECT_SOFTMAX.

Referenced by run_preference_semantics().

{ 
 const int exploration_policy = exploration_get_policy( my_agent );
 preference *return_val = NULL;

 const bool my_rl_enabled = rl_enabled( my_agent );

 const rl_param_container::learning_choices my_learning_policy = my_rl_enabled ? my_agent->rl_params->learning_policy->get_value() : rl_param_container::q;

 // get preference values for each candidate
 // see soar_ecPrintPreferences
 for ( preference *cand = candidates; cand; cand = cand->next_candidate )
  exploration_compute_value_of_candidate( my_agent, cand, s );

 double top_value = candidates->numeric_value;
 bool top_rl = candidates->rl_contribution;

 // should find highest valued candidate in q-learning
 if ( my_rl_enabled && my_learning_policy == rl_param_container::q )
 {
  for ( const preference * cand = candidates; cand; cand = cand->next_candidate )
  {
   if ( cand->numeric_value > top_value )
   {
    top_value = cand->numeric_value;
    top_rl = cand->rl_contribution;
   }
  }
 }

 switch ( exploration_policy )
 {
  case USER_SELECT_FIRST:
   return_val = candidates;
   break;

  case USER_SELECT_LAST:
   for ( return_val = candidates; return_val->next_candidate; return_val = return_val->next_candidate );
   break;

  case USER_SELECT_RANDOM:
   return_val = exploration_randomly_select( candidates );
   break;

  case USER_SELECT_SOFTMAX:
   return_val = exploration_probabilistically_select( candidates );
   break;

  case USER_SELECT_E_GREEDY:
   return_val = exploration_epsilon_greedy_select( my_agent, candidates );
   break;

  case USER_SELECT_BOLTZMANN:
   return_val = exploration_boltzmann_select( my_agent, candidates );
   break;
 }

 // should perform update here for chosen candidate in sarsa 
 if ( my_rl_enabled )
 {
  rl_tabulate_reward_values( my_agent );

  if ( my_learning_policy == rl_param_container::sarsa )
  {
   rl_perform_update( my_agent, return_val->numeric_value, return_val->rl_contribution, s->id );
  }
  else if ( my_learning_policy == rl_param_container::q )
  {
   rl_perform_update( my_agent, top_value, top_rl, s->id );

   if ( return_val->numeric_value != top_value )
    rl_watkins_clear( my_agent, s->id );
  }
 }

 return return_val;
}

void exploration_compute_value_of_candidate	(	agent *	my_agent,
		preference *	cand,
		slot *	s,
		double	default_value
	)

Definition at line 877 of file exploration.cpp.

References BINARY_INDIFFERENT_PREFERENCE_TYPE, get_number_from_symbol(), preference_struct::next, agent_struct::numeric_indifferent_mode, NUMERIC_INDIFFERENT_MODE_AVG, NUMERIC_INDIFFERENT_PREFERENCE_TYPE, preference_struct::numeric_value, slot_struct::preferences, preference_struct::rl_contribution, preference_struct::total_preferences_for_candidate, and preference_struct::value.

Referenced by exploration_choose_according_to_policy(), exploration_probability_according_to_policy(), require_preference_semantics(), and run_preference_semantics().

{
 if ( !cand )
  return;

 // initialize candidate values
 cand->total_preferences_for_candidate = 0;
 cand->numeric_value = 0;
 cand->rl_contribution = false;

 // all numeric indifferents
 for ( preference *pref = s->preferences[ NUMERIC_INDIFFERENT_PREFERENCE_TYPE ]; pref; pref = pref->next)
 {
  if ( cand->value == pref->value )
  {
   cand->total_preferences_for_candidate += 1;
   cand->numeric_value += get_number_from_symbol( pref->referent );

   if ( pref->inst->prod->rl_rule )
   {
    cand->rl_contribution = true;
   }
  }
 }

 // all binary indifferents
 for ( preference *pref = s->preferences[ BINARY_INDIFFERENT_PREFERENCE_TYPE ]; pref; pref = pref->next )
 {
  if (cand->value == pref->value)
  {
   cand->total_preferences_for_candidate += 1;
   cand->numeric_value += get_number_from_symbol( pref->referent );
         }
 }

 // if no contributors, provide default
 if ( !cand->total_preferences_for_candidate )
 {
  cand->numeric_value = default_value;
  cand->total_preferences_for_candidate = 1;
 }

 // accomodate average mode
 if ( my_agent->numeric_indifferent_mode == NUMERIC_INDIFFERENT_MODE_AVG )
  cand->numeric_value = cand->numeric_value / cand->total_preferences_for_candidate;
}

const int exploration_convert_parameter	(	agent *	my_agent,
		const char *	name
	)

Definition at line 143 of file exploration.cpp.

References EXPLORATION_PARAMS, agent_struct::exploration_params, and exploration_parameter_struct::name.

Referenced by exploration_get_parameter_value(), exploration_get_reduction_policy(), exploration_get_reduction_rate(), exploration_set_parameter_value(), exploration_set_reduction_policy(), exploration_set_reduction_rate(), exploration_valid_parameter(), exploration_valid_parameter_value(), and exploration_valid_reduction_rate().

{
 for ( int i = 0; i < EXPLORATION_PARAMS; ++i )
  if ( !strcmp( name, my_agent->exploration_params[ i ]->name ) )
   return i;

 return EXPLORATION_PARAMS;
}

const char* exploration_convert_parameter	(	agent *	my_agent,
		const int	parameter
	)

Definition at line 152 of file exploration.cpp.

References EXPLORATION_PARAMS, agent_struct::exploration_params, and exploration_parameter_struct::name.

{
 return ( parameter >= 0 && parameter < EXPLORATION_PARAMS ) ? my_agent->exploration_params[ parameter ]->name : NULL ;
}

const int exploration_convert_policy

(

const char *

policy_name

)

Definition at line 55 of file exploration.cpp.

References USER_SELECT_BOLTZMANN, USER_SELECT_E_GREEDY, USER_SELECT_FIRST, USER_SELECT_LAST, USER_SELECT_RANDOM, and USER_SELECT_SOFTMAX.

Referenced by exploration_set_policy(), and exploration_valid_policy().

{
 if ( !strcmp( policy_name, "boltzmann" ) )
  return USER_SELECT_BOLTZMANN;
 if ( !strcmp( policy_name, "epsilon-greedy" ) )
  return USER_SELECT_E_GREEDY;
 if ( !strcmp( policy_name, "first" ) )
  return USER_SELECT_FIRST;
 if ( !strcmp( policy_name, "last" ) )
  return USER_SELECT_LAST;
 if ( !strcmp( policy_name, "random-uniform" ) )
  return USER_SELECT_RANDOM;
 if ( !strcmp( policy_name, "softmax" ) )
  return USER_SELECT_SOFTMAX;

 return 0;
}

const char* exploration_convert_policy

(

const int

policy

)

Definition at line 73 of file exploration.cpp.

References USER_SELECT_BOLTZMANN, USER_SELECT_E_GREEDY, USER_SELECT_FIRST, USER_SELECT_LAST, USER_SELECT_RANDOM, and USER_SELECT_SOFTMAX.

{
 if ( policy == USER_SELECT_BOLTZMANN )
  return "boltzmann";
 if ( policy == USER_SELECT_E_GREEDY )
  return "epsilon-greedy";
 if ( policy == USER_SELECT_FIRST )
  return "first";
 if ( policy == USER_SELECT_LAST )
  return "last";
 if ( policy == USER_SELECT_RANDOM )
  return "random-uniform";
 if ( policy == USER_SELECT_SOFTMAX )
  return "softmax";

 return NULL;
}

const int exploration_convert_reduction_policy

(

const char *

policy_name

)

Definition at line 304 of file exploration.cpp.

References EXPLORATION_REDUCTION_EXPONENTIAL, EXPLORATION_REDUCTION_LINEAR, and EXPLORATION_REDUCTIONS.

Referenced by exploration_get_reduction_rate(), exploration_set_reduction_policy(), exploration_set_reduction_rate(), exploration_valid_reduction_policy(), and exploration_valid_reduction_rate().

{
 if ( !strcmp( policy_name, "exponential" ) )
  return EXPLORATION_REDUCTION_EXPONENTIAL;
 if ( !strcmp( policy_name, "linear" ) )
  return EXPLORATION_REDUCTION_LINEAR;

 return EXPLORATION_REDUCTIONS;
}

const char* exploration_convert_reduction_policy

(

const int

policy

)

Definition at line 314 of file exploration.cpp.

References EXPLORATION_REDUCTION_EXPONENTIAL, and EXPLORATION_REDUCTION_LINEAR.

{
 if ( policy == EXPLORATION_REDUCTION_EXPONENTIAL )
  return "exponential";
 if ( policy == EXPLORATION_REDUCTION_LINEAR )
  return "linear";

 return NULL;
}

preference* exploration_epsilon_greedy_select	(	agent *	my_agent,
		preference *	candidates
	)

Definition at line 808 of file exploration.cpp.

References exploration_get_highest_q_value_pref(), exploration_get_parameter_value(), EXPLORATION_PARAM_EPSILON, exploration_randomly_select(), preference_struct::next_candidate, print(), print_with_symbols(), SoarRand(), agent_struct::sysparams, TRACE_INDIFFERENT_SYSPARAM, xml_att_val(), xml_begin_tag(), and xml_end_tag().

Referenced by exploration_choose_according_to_policy().

{
 const double epsilon = exploration_get_parameter_value( my_agent, EXPLORATION_PARAM_EPSILON );

 if ( my_agent->sysparams[ TRACE_INDIFFERENT_SYSPARAM ] )
 {
  for ( const preference *cand = candidates; cand; cand = cand->next_candidate )
  {
   print_with_symbols( my_agent, "\n Candidate %y:  ", cand->value );
   print( my_agent, "Value (Sum) = %f", cand->numeric_value );
   xml_begin_tag( my_agent, kTagCandidate );
   xml_att_val( my_agent, kCandidateName, cand->value );
   xml_att_val( my_agent, kCandidateType, kCandidateTypeSum );
   xml_att_val( my_agent, kCandidateValue, cand->numeric_value );
   xml_end_tag( my_agent, kTagCandidate );
  }
 }

 if ( SoarRand() < epsilon )
  return exploration_randomly_select( candidates );
 else
  return exploration_get_highest_q_value_pref( candidates );
}

bool exploration_get_auto_update

(

agent *

my_agent

)

Definition at line 254 of file exploration.cpp.

References FALSE, agent_struct::sysparams, and USER_SELECT_REDUCE_SYSPARAM.

Referenced by exploration_update_parameters().

{
 return my_agent->sysparams[ USER_SELECT_REDUCE_SYSPARAM ] != FALSE;
}

preference* exploration_get_highest_q_value_pref

(

preference *

candidates

)

Definition at line 835 of file exploration.cpp.

References preference_struct::next_candidate, preference_struct::numeric_value, SoarRandInt(), and uint32_t().

Referenced by exploration_epsilon_greedy_select().

{
 preference *top_cand = candidates;
 double top_value = candidates->numeric_value;
 int num_max_cand = 0;

 for ( preference * cand = candidates; cand; cand = cand->next_candidate )
 {
  if ( cand->numeric_value > top_value )
  {
   top_value = cand->numeric_value;
   top_cand = cand;
   num_max_cand = 1;
  } 
  else if ( cand->numeric_value == top_value )
   ++num_max_cand;
 }

 if ( num_max_cand == 1 )
  return top_cand;
 else
 {
  preference *cand = candidates;
  while ( cand->numeric_value != top_value )
   cand = cand->next_candidate;

  // if operators tied for highest Q-value, select among tied set at random
  for ( uint32_t chosen_num = SoarRandInt( num_max_cand - 1 ); chosen_num; --chosen_num )
  {
   cand = cand->next_candidate;

   while ( cand->numeric_value != top_value )
    cand = cand->next_candidate;
  }

  return cand;
 }
}

double exploration_get_parameter_value	(	agent *	my_agent,
		const char *	parameter
	)

Definition at line 173 of file exploration.cpp.

References exploration_convert_parameter(), EXPLORATION_PARAMS, agent_struct::exploration_params, and exploration_parameter_struct::value.

Referenced by exploration_boltzmann_select(), exploration_epsilon_greedy_select(), exploration_probability_according_to_policy(), and exploration_update_parameters().

{ 
 const int param = exploration_convert_parameter( my_agent, parameter );
 if ( param == EXPLORATION_PARAMS )
  return 0;

 return my_agent->exploration_params[ param ]->value;
}

double exploration_get_parameter_value	(	agent *	my_agent,
		const int	parameter
	)

Definition at line 182 of file exploration.cpp.

References agent_struct::exploration_params, exploration_valid_parameter(), and exploration_parameter_struct::value.

{
 if ( exploration_valid_parameter( my_agent, parameter ) )
  return my_agent->exploration_params[ parameter ]->value;

 return 0;
}

const int exploration_get_policy

(

agent *

my_agent

)

Definition at line 118 of file exploration.cpp.

References agent_struct::sysparams, and USER_SELECT_MODE_SYSPARAM.

Referenced by exploration_choose_according_to_policy(), and exploration_probability_according_to_policy().

{
 return static_cast<int>(my_agent->sysparams[ USER_SELECT_MODE_SYSPARAM ]);
}

const int exploration_get_reduction_policy	(	agent *	my_agent,
		const char *	parameter
	)

Definition at line 327 of file exploration.cpp.

References exploration_convert_parameter(), EXPLORATION_PARAMS, agent_struct::exploration_params, EXPLORATION_REDUCTIONS, and exploration_parameter_struct::reduction_policy.

Referenced by exploration_update_parameters().

{
 const int param = exploration_convert_parameter( my_agent, parameter );
 if ( param == EXPLORATION_PARAMS )
  return EXPLORATION_REDUCTIONS;

 return my_agent->exploration_params[ param ]->reduction_policy;
}

const int exploration_get_reduction_policy	(	agent *	my_agent,
		const int	parameter
	)

Definition at line 336 of file exploration.cpp.

References agent_struct::exploration_params, EXPLORATION_REDUCTIONS, exploration_valid_parameter(), and exploration_parameter_struct::reduction_policy.

{
 if ( exploration_valid_parameter( my_agent, parameter ) )
  return my_agent->exploration_params[ parameter ]->reduction_policy;
 else
  return EXPLORATION_REDUCTIONS;
}

double exploration_get_reduction_rate	(	agent *	my_agent,
		const char *	parameter,
		const char *	policy_name
	)

Definition at line 449 of file exploration.cpp.

References exploration_convert_parameter(), exploration_convert_reduction_policy(), exploration_get_reduction_rate(), EXPLORATION_PARAMS, and EXPLORATION_REDUCTIONS.

Referenced by exploration_get_reduction_rate(), and exploration_update_parameters().

{
 const int param = exploration_convert_parameter( my_agent, parameter );
 if ( param == EXPLORATION_PARAMS )
  return 0;

 const int policy = exploration_convert_reduction_policy( policy_name );
 if ( policy == EXPLORATION_REDUCTIONS )
  return 0;

 return exploration_get_reduction_rate( my_agent, param, policy );
}

double exploration_get_reduction_rate	(	agent *	my_agent,
		const int	parameter,
		const int	policy
	)

Definition at line 462 of file exploration.cpp.

References agent_struct::exploration_params, exploration_valid_parameter(), exploration_valid_reduction_policy(), and exploration_parameter_struct::rates.

{ 
 if ( exploration_valid_parameter( my_agent, parameter ) &&
   exploration_valid_reduction_policy( my_agent, parameter, policy ) )
  return my_agent->exploration_params[ parameter ]->rates[ policy ];

 return 0;
}

preference* exploration_probabilistically_select

(

preference *

candidates

)

Definition at line 700 of file exploration.cpp.

References exploration_randomly_select(), preference_struct::next_candidate, NIL, preference_struct::numeric_value, and SoarRand().

Referenced by exploration_choose_according_to_policy().

{
 // IF THIS FUNCTION CHANGES, SEE soar_ecPrintPreferences

 // count up positive numbers
 double total_probability = 0.0;
 for ( const preference *cand = candidates; cand; cand = cand->next_candidate )
  if ( cand->numeric_value > 0 )
   total_probability += cand->numeric_value;

 // if nothing positive, resort to random
 if ( total_probability == 0.0 )
  return exploration_randomly_select( candidates );

 // choose a random preference within the distribution
 const double selected_probability = total_probability * SoarRand();

 // select the candidate based upon the chosen preference
 double current_sum = 0.0;
 for ( preference *cand = candidates; cand; cand = cand->next_candidate )
 {
  if ( cand->numeric_value > 0 )
  {
   current_sum += cand->numeric_value;
   if ( selected_probability <= current_sum )
    return cand;
  }
 }

 return NIL;
}

double exploration_probability_according_to_policy	(	agent *	my_agent,
		slot *	s,
		preference *	candidates,
		preference *	selection
	)

Definition at line 584 of file exploration.cpp.

References exploration_compute_value_of_candidate(), exploration_get_parameter_value(), exploration_get_policy(), EXPLORATION_PARAM_EPSILON, EXPLORATION_PARAM_TEMPERATURE, preference_struct::next_candidate, preference_struct::numeric_value, USER_SELECT_BOLTZMANN, USER_SELECT_E_GREEDY, USER_SELECT_FIRST, USER_SELECT_LAST, USER_SELECT_RANDOM, and USER_SELECT_SOFTMAX.

{ 
  const int exploration_policy = exploration_get_policy(my_agent);

  // get preference values for each candidate
  // see soar_ecPrintPreferences
  for(preference *cand = candidates; cand; cand = cand->next_candidate)
    exploration_compute_value_of_candidate(my_agent, cand, s);

  switch(exploration_policy)
  {
    case USER_SELECT_FIRST:
      return candidates == selection ? 1.0f : 0.0f;

    case USER_SELECT_LAST:
      return selection->next_candidate ? 0.0f: 1.0f;

    case USER_SELECT_RANDOM:
    {
      unsigned int cand_count = 0;
      for(const preference * cand = candidates; cand; cand = cand->next_candidate)
        ++cand_count;
      return 1.0 / cand_count;
    }

    case USER_SELECT_SOFTMAX:
    {
      unsigned int cand_count = 0;
      double total_probability = 0.0;
      for(const preference *cand = candidates; cand; cand = cand->next_candidate) {
        ++cand_count;
        if(cand->numeric_value > 0)
          total_probability += cand->numeric_value;
      }

      if(total_probability > 0) {
        if(selection->numeric_value > 0)
          return selection->numeric_value / total_probability;
        else
          return 0.0;
      }
      else
        return 1.0 / cand_count;
    }

    case USER_SELECT_E_GREEDY:
    {
      const double epsilon = exploration_get_parameter_value(my_agent, EXPLORATION_PARAM_EPSILON);

      double top_value = candidates->numeric_value;
      unsigned int top_count = 0;
      unsigned int cand_count = 0;
      for(const preference * cand = candidates; cand; cand = cand->next_candidate) {
        ++cand_count;
        if(cand->numeric_value > top_value) {
          top_value = cand->numeric_value;
          top_count = 1;
        }
        else if(cand->numeric_value == top_value)
          ++top_count;
      }

      double retval = epsilon / cand_count;
      if(selection->numeric_value == top_value)
        retval += (1.0 - epsilon) / top_count;
      return retval;
    }

    case USER_SELECT_BOLTZMANN:
    {
      const double t = exploration_get_parameter_value(my_agent, EXPLORATION_PARAM_TEMPERATURE);

      double maxq = candidates->numeric_value;
      for(preference *cand = candidates->next_candidate; cand; cand = cand->next_candidate) {
        if(maxq < cand->numeric_value)
          maxq = cand->numeric_value;
      }

      double exptotal = 0.0;
      double expselection = 0.0;
      for(preference *cand = candidates; cand; cand = cand->next_candidate) {
        // equivalent to exp((cand->numeric_value / t) - (maxq / t)) but safer against overflow
        double v = exp((cand->numeric_value - maxq) / t);
        exptotal += v;
        if(cand == selection)
          expselection = v;
      }

      return expselection / exptotal;
    }

    default:
      abort();
      return 0.0;
  }
}

preference* exploration_randomly_select

(

preference *

candidates

)

Definition at line 684 of file exploration.cpp.

References preference_struct::next_candidate, SoarRandInt(), and uint32_t().

Referenced by exploration_choose_according_to_policy(), exploration_epsilon_greedy_select(), and exploration_probabilistically_select().

{
 unsigned int cand_count = 0;
 for ( const preference * cand = candidates; cand; cand = cand->next_candidate )
  ++cand_count;

 preference * cand = candidates;
 for( uint32_t chosen_num = SoarRandInt( cand_count - 1 ); chosen_num; --chosen_num )
  cand = cand->next_candidate;

 return cand;
}

bool exploration_set_auto_update	(	agent *	my_agent,
		bool	setting
	)

Definition at line 262 of file exploration.cpp.

References FALSE, agent_struct::sysparams, TRUE, and USER_SELECT_REDUCE_SYSPARAM.

{
 my_agent->sysparams[ USER_SELECT_REDUCE_SYSPARAM ] = setting ? TRUE : FALSE;

 return true;
}

bool exploration_set_parameter_value	(	agent *	my_agent,
		const char *	name,
		double	value
	)

Definition at line 229 of file exploration.cpp.

References exploration_convert_parameter(), EXPLORATION_PARAMS, agent_struct::exploration_params, and exploration_parameter_struct::value.

Referenced by exploration_update_parameters().

{
 const int param = exploration_convert_parameter( my_agent, name );
 if ( param == EXPLORATION_PARAMS )
  return false;

 my_agent->exploration_params[ param ]->value = value;

 return true;
}

bool exploration_set_parameter_value	(	agent *	my_agent,
		const int	parameter,
		double	value
	)

Definition at line 240 of file exploration.cpp.

References agent_struct::exploration_params, exploration_valid_parameter(), and exploration_parameter_struct::value.

{
 if ( exploration_valid_parameter( my_agent, parameter ) )
 {
  my_agent->exploration_params[ parameter ]->value = value;
  return true;
 }
 else
  return false;
}

bool exploration_set_policy	(	agent *	my_agent,
		const char *	policy_name
	)

Definition at line 94 of file exploration.cpp.

References exploration_convert_policy(), and exploration_set_policy().

Referenced by exploration_set_policy().

{ 
 const int policy = exploration_convert_policy( policy_name );

 if ( policy )
  return exploration_set_policy( my_agent, policy );

 return false;
}

bool exploration_set_policy	(	agent *	my_agent,
		const int	policy
	)

Definition at line 104 of file exploration.cpp.

References exploration_valid_policy(), set_sysparam(), and USER_SELECT_MODE_SYSPARAM.

{ 
 if ( exploration_valid_policy( policy ) )
 {
  set_sysparam( my_agent, USER_SELECT_MODE_SYSPARAM, policy );
  return true;
 }
 
 return false;
}

bool exploration_set_reduction_policy	(	agent *	my_agent,
		const char *	parameter,
		const char *	policy_name
	)

Definition at line 365 of file exploration.cpp.

References exploration_convert_parameter(), exploration_convert_reduction_policy(), EXPLORATION_PARAMS, agent_struct::exploration_params, EXPLORATION_REDUCTIONS, and exploration_parameter_struct::reduction_policy.

{
 const int param = exploration_convert_parameter( my_agent, parameter );
 if ( param == EXPLORATION_PARAMS )
  return false;

 const int policy = exploration_convert_reduction_policy( policy_name );
 if ( policy == EXPLORATION_REDUCTIONS )
  return false;

 my_agent->exploration_params[ param ]->reduction_policy = policy;

 return true;
}

bool exploration_set_reduction_policy	(	agent *	my_agent,
		const int	parameter,
		const int	policy
	)

Definition at line 380 of file exploration.cpp.

References agent_struct::exploration_params, exploration_valid_parameter(), exploration_valid_reduction_policy(), and exploration_parameter_struct::reduction_policy.

{
 if ( exploration_valid_parameter( my_agent, parameter ) &&
   exploration_valid_reduction_policy( my_agent, parameter, policy ) )
 {
  my_agent->exploration_params[ parameter ]->reduction_policy = policy;
  return true;
 }

 return false;
}

bool exploration_set_reduction_rate	(	agent *	my_agent,
		const char *	parameter,
		const char *	policy_name,
		double	reduction_rate
	)

Definition at line 474 of file exploration.cpp.

References exploration_convert_parameter(), exploration_convert_reduction_policy(), EXPLORATION_PARAMS, EXPLORATION_REDUCTIONS, and exploration_set_reduction_rate().

Referenced by exploration_set_reduction_rate().

{
 const int param = exploration_convert_parameter( my_agent, parameter );
 if ( param == EXPLORATION_PARAMS )
  return false;

 const int policy = exploration_convert_reduction_policy( policy_name );
 if ( policy == EXPLORATION_REDUCTIONS )
  return false;

 return exploration_set_reduction_rate( my_agent, param, policy, reduction_rate );
}

bool exploration_set_reduction_rate	(	agent *	my_agent,
		const int	parameter,
		const int	policy,
		double	reduction_rate
	)

Definition at line 487 of file exploration.cpp.

References agent_struct::exploration_params, exploration_valid_parameter(), exploration_valid_reduction_policy(), exploration_valid_reduction_rate(), and exploration_parameter_struct::rates.

{
 if ( exploration_valid_parameter( my_agent, parameter ) &&
   exploration_valid_reduction_policy( my_agent, parameter, policy ) &&
   exploration_valid_reduction_rate( my_agent, parameter, policy, reduction_rate ) )
 {
  my_agent->exploration_params[ parameter ]->rates[ policy ] = reduction_rate;
  return true;
 }

 return false;
}

void exploration_update_parameters

(

agent *

my_agent

)

Definition at line 272 of file exploration.cpp.

References exploration_get_auto_update(), exploration_get_parameter_value(), exploration_get_reduction_policy(), exploration_get_reduction_rate(), EXPLORATION_PARAMS, EXPLORATION_REDUCTION_EXPONENTIAL, EXPLORATION_REDUCTION_LINEAR, and exploration_set_parameter_value().

Referenced by do_decision_phase().

{ 
 if ( exploration_get_auto_update( my_agent ) )
 {
  for ( int i = 0; i < EXPLORATION_PARAMS; ++i )
  {
   const int reduction_policy = exploration_get_reduction_policy( my_agent, i );
   const double reduction_rate = exploration_get_reduction_rate( my_agent, i, reduction_policy );

   if ( reduction_policy == EXPLORATION_REDUCTION_EXPONENTIAL )
   {
    if ( reduction_rate != 1 )
    {
     const double current_value = exploration_get_parameter_value( my_agent, i );

     exploration_set_parameter_value( my_agent, i, current_value * reduction_rate );
    }
   }
   else if ( reduction_policy == EXPLORATION_REDUCTION_LINEAR )
   {
    const double current_value = exploration_get_parameter_value( my_agent, i );

    if ( current_value > 0 && reduction_rate != 0.0 )
     exploration_set_parameter_value( my_agent, i, (current_value - reduction_rate > 0) ? (current_value - reduction_rate) : 0 );
   }
  }
 }
}

bool exploration_valid_exponential

(

double

reduction_rate

)

Definition at line 433 of file exploration.cpp.

Referenced by exploration_valid_reduction_rate().

{
 return reduction_rate >= 0 && reduction_rate <= 1;
}

bool exploration_valid_linear

(

double

reduction_rate

)

Definition at line 441 of file exploration.cpp.

Referenced by exploration_valid_reduction_rate().

{
 return reduction_rate >= 0;
}

const bool exploration_valid_parameter	(	agent *	my_agent,
		const char *	name
	)

Definition at line 160 of file exploration.cpp.

References exploration_convert_parameter(), and EXPLORATION_PARAMS.

Referenced by exploration_get_parameter_value(), exploration_get_reduction_policy(), exploration_get_reduction_rate(), exploration_set_parameter_value(), exploration_set_reduction_policy(), exploration_set_reduction_rate(), and exploration_valid_parameter_value().

{
 return exploration_convert_parameter( my_agent, name ) != EXPLORATION_PARAMS;
}

const bool exploration_valid_parameter	(	agent *	my_agent,
		const int	parameter
	)

Definition at line 165 of file exploration.cpp.

References exploration_convert_parameter().

{
 return exploration_convert_parameter( my_agent, parameter ) != NULL;
}

bool exploration_valid_parameter_value	(	agent *	my_agent,
		const char *	name,
		double	value
	)

Definition at line 209 of file exploration.cpp.

References exploration_convert_parameter(), EXPLORATION_PARAMS, agent_struct::exploration_params, and exploration_parameter_struct::val_func.

{
 const int param = exploration_convert_parameter( my_agent, name );
 if ( param == EXPLORATION_PARAMS )
  return false;

 return my_agent->exploration_params[ param ]->val_func( value );
}

bool exploration_valid_parameter_value	(	agent *	my_agent,
		const int	parameter,
		double	value
	)

Definition at line 218 of file exploration.cpp.

References agent_struct::exploration_params, exploration_valid_parameter(), and exploration_parameter_struct::val_func.

{
 if ( exploration_valid_parameter( my_agent, parameter ) )
  return my_agent->exploration_params[ parameter ]->val_func( value );

 return false;
}

bool exploration_valid_policy

(

const char *

policy_name

)

Definition at line 42 of file exploration.cpp.

References exploration_convert_policy().

Referenced by exploration_set_policy().

{
 return exploration_convert_policy( policy_name ) != 0;
}

bool exploration_valid_policy

(

const int

policy

)

Definition at line 47 of file exploration.cpp.

References USER_SELECT_INVALID.

{
 return policy > 0 && policy < USER_SELECT_INVALID;
}

bool exploration_valid_reduction_policy	(	agent *	,
		const char *	,
		const char *	policy_name
	)

Definition at line 347 of file exploration.cpp.

References exploration_convert_reduction_policy(), and EXPLORATION_REDUCTIONS.

Referenced by exploration_get_reduction_rate(), exploration_set_reduction_policy(), exploration_set_reduction_rate(), and exploration_valid_reduction_rate().

{ 
 return exploration_convert_reduction_policy( policy_name ) != EXPLORATION_REDUCTIONS;
}

bool exploration_valid_reduction_policy	(	agent *	,
		const char *	,
		const int	policy
	)

Definition at line 352 of file exploration.cpp.

References exploration_convert_reduction_policy().

{ 
 return exploration_convert_reduction_policy( policy ) != NULL;
}

bool exploration_valid_reduction_policy	(	agent *	,
		const int	,
		const int	policy
	)

Definition at line 357 of file exploration.cpp.

References exploration_convert_reduction_policy().

{ 
 return exploration_convert_reduction_policy( policy ) != NULL;
}

bool exploration_valid_reduction_rate	(	agent *	my_agent,
		const char *	parameter,
		const char *	policy_name,
		double	reduction_rate
	)

Definition at line 395 of file exploration.cpp.

References exploration_convert_parameter(), exploration_convert_reduction_policy(), EXPLORATION_PARAMS, EXPLORATION_REDUCTIONS, and exploration_valid_reduction_rate().

Referenced by exploration_set_reduction_rate(), and exploration_valid_reduction_rate().

{
 const int param = exploration_convert_parameter( my_agent, parameter );
 if ( param == EXPLORATION_PARAMS )
  return false;

 const int policy = exploration_convert_reduction_policy( policy_name );
 if ( policy == EXPLORATION_REDUCTIONS )
  return false;

 return exploration_valid_reduction_rate( my_agent, param, policy, reduction_rate );
}

bool exploration_valid_reduction_rate	(	agent *	my_agent,
		const int	parameter,
		const int	policy,
		double	reduction_rate
	)

Definition at line 408 of file exploration.cpp.

References EXPLORATION_REDUCTION_EXPONENTIAL, EXPLORATION_REDUCTION_LINEAR, exploration_valid_exponential(), exploration_valid_linear(), and exploration_valid_reduction_policy().

{
 if ( !exploration_valid_reduction_policy( my_agent, parameter, policy ) )
  return false;

 switch ( policy )
 {
  case EXPLORATION_REDUCTION_EXPONENTIAL:
   return exploration_valid_exponential( reduction_rate );
   break;

  case EXPLORATION_REDUCTION_LINEAR:
   return exploration_valid_linear( reduction_rate );
   break;

  default:
   break;
 }

 return false;
}

bool exploration_validate_epsilon

(

double

value

)

Definition at line 193 of file exploration.cpp.

Referenced by create_soar_agent().

{
 return value >= 0 && value <= 1;
}

bool exploration_validate_temperature

(

double

value

)

Definition at line 201 of file exploration.cpp.

Referenced by create_soar_agent().

{
 return value > 0;
}