;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;  experimentation.lisp
;;;  Written by:  Shaul Markovitch
;;;  Last time modified: 10/1/2000
;;;
;;;
;;;Included functions:
;;;
;;;
;;; This files contains the code  that handle experimentation with the
;;; system.  
;;;
;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defparameter *plot-learning-curve* t
  "Controls whether to use gnuplot to plot the learning curves")


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; test-learning
;;;
;;; This is the main experimentation function.  Its arguments are the
;;; learning strategy (it is a "learning" structure), a set of test
;;; problems. A number of testing points rqeuired or a list of testing
;;; points. and a flag that specifies whether a plot should be drawn
;;; at the end.  
;;; It returns a structure of the type "learning-outcome" which
;;; includes the set of macros learned, a structure that contains
;;; information about the resources used during learning and
;;; a list of outcomes of the tests during learning.
;;;
;;; examples:
;;;   (test-learning)
;;;   will generate a set of 100 problems (depends on the variable
;;;   *default-test-set-size*), will use the default learning
;;;   strategy, will conduct a learning session of 1000 problems
;;;   (again, depends on the defaults), and will perform one test
;;;   before learning and then tests after 200, 400, 600, 800, 1000
;;;   problems (the default number of tests is 5).
;;;
;;;   (test-learning
;;;          :learning-strategy (make-learning :max-resources 100)
;;;          :testing-points '(5 10 25 50 100))
;;;   will learn on 100 problems performing tests in the specified
;;;   points.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun test-learning
  (&key
   (learning-strategies (list (make-learning)))
   (test-set (loop repeat *default-test-set-size*
			collect (generate-problem
				 (learning-domain (first learning-strategies))
				 (learning-generation-filtering
				   (first learning-strategies)))
				 ))
   (testing-points (learning-testing-points (first learning-strategies)))
   (plot-learning-curve-flag *plot-learning-curve*)
   (eps-flag nil)
   (plot-labels nil)
   (plot-file-name nil)
   &aux results)
  (setf results
	(loop for learning-strategy in
	      learning-strategies
	      collect
	      (learn-macros :learning-strategy learning-strategy
		:test-set test-set
		:test-points
		(if (numberp testing-points)
		    (append
		     (loop with n = (floor (/ (learning-max-resources
					      learning-strategy)
					     testing-points))
			  for i from n by n repeat (1- testing-points)
			  collect i)
		     (list (learning-max-resources
			    learning-strategy)))
		  testing-points))))
			  
  (when plot-learning-curve-flag
	(plot-learning-curve results
	:plot-labels plot-labels
	:plot-file-name plot-file-name
	:eps-flag eps-flag))
  results)
  
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; create-test-set
;;;
;;; creates a set of a testing problems.  
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defun create-test-set
  (&key
   (learning-strategy *default-learning-strategy*)
   (size  *default-test-set-size*))
  (loop repeat size
	collect (generate-problem
		 (learning-domain learning-strategy)
		
		 (learning-generation-filtering
		  learning-strategy)
		 )))



;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; plot-learning-curve
;;;
;;; This function utilizes the "generate-plot" function (found in the
;;; "utilities.lisp" file) to plot the results of a learning experiment.
;;;  Its main input is the list of test results.  Each test result 
;;; is a pair.  The first element is a "learning-resources" structure 
;;; which specifies the learning resource consumed to this point
;;; and the second element is a "problem-set-outcome" structure which
;;; stores mean numbers for various measurement, such as generated 
;;; nodes, expanded nodes etc.  Each of this results is given
;;; as a statistics record.  At this point we don't use the stds but 
;;; later we will add this feature to the function "generate-plot".
;;; 
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defparameter *default-plot-x* 'problems)

(defun plot-learning-curve
  (results
   &key
   (eps-flag nil)
   (plot-file-name nil)
   (plot-labels nil))
  		      
  (let ((x-list (loop for res in (learning-outcome-test-results
				  (first results))
		      collect (case *default-plot-x*
				    (problems 
				     (learning-resources-problems
				      (first res)))
				    (macros  (learning-resources-macros
				      (first res))))))
	(gen-lists (loop for r in results
			 collect
			 (loop for outcome in
			       (learning-outcome-test-results r)
			       collect
			       (statistics-mean (problem-set-outcome-generated
			       (second outcome))))))

	(length-lists (loop for r in results
			 collect
			 (loop for outcome in
			       (learning-outcome-test-results r)
			       collect
			       (statistics-mean
				(problem-set-outcome-solution-length
			       (second outcome)))))))
    (generate-plot x-list gen-lists
		   :x-label *default-plot-x*
		   :y-label "generated"
		   :plot-labels plot-labels
		   :eps-flag eps-flag
		   :filename (concatenate 'string
					   plot-file-name "-gen")
		   )
    
    (generate-plot x-list length-lists
		   :x-label "problems"
		   :plot-labels plot-labels
		   :y-label "solution length"
		   :eps-flag eps-flag
		   :filename  (concatenate 'string
					   plot-file-name "-len")
		   )))


(defun create-problem-set
  (&key
   (domain *default-domain*)
   (size  *default-test-set-size*)
   (generation-method  *default-generation-filtering-technique*))
  (loop repeat size
	collect (generate-problem domain
				  generation-method)))

(defparameter *default-compare-p-to-p-eps-flag* nil)
(defun compare-peak-to-peak ()
  (test-learning :learning-strategies
		 (list
		  (make-learning :attention-filtering
				(make-instance 'min-to-better))
		 (make-learning :attention-filtering
				(make-instance 'peak-to-peak)))
		 :plot-learning-curve-flag t
		 :eps-flag *default-compare-p-to-p-eps-flag*
		 :plot-labels (list "min-to-better" "peak-to-peak")
		 :plot-file-name "p-to-p-min-to-b"))