optimize: added Powell and Nelder-Mead method

optimize.cc

@@ -233,6 +233,112 @@ return optresult<T>(f,converged);
 }
 
 
+//function along the line x0 + t*u for linemin_brent, keeps the best point and counts the evaluations
+template<typename T>
+class linefunction {
+public:
+optevaluator<T> &e;
+const NRVec<T> &x0;
+const NRVec<T> &u;
+int maxiter; //evaluation budget
+int nev; //evaluations done
+T tbest;
+T fbest;
+linefunction(optevaluator<T> &ee, const NRVec<T> &xx0, const T f0, const NRVec<T> &uu, const int maxit) : e(ee), x0(xx0), u(uu), maxiter(std::min(maxit,ee.remaining())), nev(0), tbest(0), fbest(f0) {};
+NRVec<T> point(const T t) const {NRVec<T> x(x0); x.axpy(t,u); return x;}
+T operator()(const T t) {T f=e(point(t)); ++nev; if(f<fbest) {fbest=f; tbest=t;} return f;}
+bool exhausted() const {return nev>=maxiter;}
+};
+
+
+template<typename T>
+T linemin_brent(optevaluator<T> &e, NRVec<T> &x, T &f, const NRVec<T> &u, const T t1, const T tol, const T abstol, const int maxiter, const int verbose)
+{
+const T golden=1.618033988749895; //expansion factor of the bracketing
+const T cgold=0.3819660112501051; //golden section fraction (3-sqrt(5))/2
+linefunction<T> phi(e,x,f,u,maxiter);
+
+//bracketing: a,b,c with f(b)<=f(a), f(b)<=f(c)
+T a=0, fa=f;
+T b=t1, fb;
+T c=0, fc=0;
+bool bracketed=false;
+if(phi.exhausted()) return 0;
+fb=phi(b);
+if(fb>fa) //uphill, try the opposite direction
+	{
+	if(phi.exhausted()) return 0;
+	c= -t1; fc=phi(c);
+	if(fc>=fa) {std::swap(a,b); std::swap(fa,fb); bracketed=true;} //minimum between -t1 and t1, middle point is 0
+	else {b=c; fb=fc;}
+	}
+if(!bracketed) //expand downhill from a through b
+	{
+	for(;;)
+		{
+		if(phi.exhausted()) break;
+		c= b+golden*(b-a); fc=phi(c);
+		if(fc>=fb) {bracketed=true; break;}
+		a=b; fa=fb; b=c; fb=fc;
+		}
+	}
+if(verbose>1) std::cout <<"linemin bracket "<<a<<" "<<b<<" "<<c<<" f= "<<fa<<" "<<fb<<" "<<fc<<std::endl;
+
+//Brent's method in [lo,hi]: xm = best point, w = second best, v = previous value of w
+if(bracketed)
+	{
+	T lo=std::min(a,c), hi=std::max(a,c);
+	T xm=b, w=b, v=b, fx=fb, fw=fb, fv=fb;
+	T d=0, dprev=0; //last step and the one before
+	const T tolabs=std::max(abstol,std::numeric_limits<T>::epsilon()*std::abs(t1)); //nonzero to guarantee termination
+	while(!phi.exhausted())
+		{
+		T mid=(lo+hi)/2;
+		T tol1=tol*std::abs(xm)+tolabs;
+		T tol2=2*tol1;
+		if(std::abs(xm-mid) <= tol2-(hi-lo)/2) break; //converged
+		bool goldenstep=true;
+		if(std::abs(dprev)>tol1) //try parabolic interpolation through xm, w, v
+			{
+			T r=(xm-w)*(fx-fv);
+			T q=(xm-v)*(fx-fw);
+			T p=(xm-v)*q-(xm-w)*r;
+			q=2*(q-r);
+			if(q>0) p= -p; else q= -q;
+			if(std::abs(p)<std::abs(q*dprev/2) && p>q*(lo-xm) && p<q*(hi-xm)) //acceptable: inside the interval and less than half of the step before last
+				{
+				dprev=d;
+				d=p/q;
+				T t=xm+d;
+				if(t-lo<tol2 || hi-t<tol2) d= mid>xm ? tol1 : -tol1;
+				goldenstep=false;
+				}
+			}
+		if(goldenstep) {dprev= xm>=mid ? lo-xm : hi-xm; d=cgold*dprev;}
+		T t= std::abs(d)>=tol1 ? xm+d : xm+(d>0 ? tol1 : -tol1);
+		T ft=phi(t);
+		if(ft<=fx)
+			{
+			if(t>=xm) lo=xm; else hi=xm;
+			v=w; fv=fw; w=xm; fw=fx; xm=t; fx=ft;
+			}
+		else
+			{
+			if(t<xm) lo=t; else hi=t;
+			if(ft<=fw || w==xm) {v=w; fv=fw; w=t; fw=ft;}
+			else if(ft<=fv || v==xm || v==w) {v=t; fv=ft;}
+			}
+		}
+	}
+
+if(verbose>1) std::cout <<"linemin t= "<<phi.tbest<<" f= "<<phi.fbest<<" evaluations "<<phi.nev<<std::endl;
+if(phi.tbest==0) return 0;
+x=phi.point(phi.tbest);
+f=phi.fbest;
+return phi.tbest;
+}
+
+
 template<typename T>
 optresult<T> linesearch_minimize(const char *name, linesearch_direction<T> &dir, const OptProblem<T> *o, NRVec<T> &x, const linesearch_hyperparameters<T> &hp, hyperparameters<T> *achieved)
 {
@@ -442,6 +548,199 @@ return linesearch_minimize(names[hp.my_beta],dir,o,x,hp,achieved);
 
 
 
+//simplex of n+1 vertices with their function values
+template<typename T>
+class NM_simplex {
+public:
+int n;
+NRVec<NRVec<T> > v; //vertices
+NRVec<T> f; //function values
+NRVec<T> sum; //sum of the vertices
+int best, second, worst; //indices of the lowest, second highest and highest function value
+NM_simplex(const NRVec<T> &x0, const T h, optevaluator<T> &e);
+void order();
+void sumup() {sum=v[0]; for(int i=1; i<=n; ++i) sum+=v[i];}
+void replace(const int i, const NRVec<T> &x, const T fx) {sum-=v[i]; sum+=x; v[i]=x; f[i]=fx;}
+NRVec<T> centroid() const {NRVec<T> c(sum); c-=v[worst]; c/=(T)n; return c;} //of all vertices except the worst one
+NRVec<T> point(const NRVec<T> &c, const T t) const {NRVec<T> x = c*(1+t); x.axpy(-t,v[worst]); return x;} //c + t*(c - x_worst)
+T diameter() const;
+void shrink(const T sigma, optevaluator<T> &e);
+};
+
+//initial simplex x0, x0 + h e_i
+template<typename T>
+NM_simplex<T>::NM_simplex(const NRVec<T> &x0, const T h, optevaluator<T> &e) : n(x0.size()), v(n+1), f(n+1), best(0), second(0), worst(0)
+{
+v[0]=x0;
+f[0]=e(x0);
+for(int i=1; i<=n; ++i)
+	{
+	v[i]=x0;
+	v[i].copyonwrite();
+	v[i][i-1] += h;
+	f[i]=e(v[i]);
+	}
+sumup();
+}
+
+template<typename T>
+void NM_simplex<T>::order()
+{
+best=worst=0;
+for(int i=1; i<=n; ++i)
+	{
+	if(f[i]<f[best]) best=i;
+	if(f[i]>=f[worst]) worst=i;
+	}
+second=best;
+for(int i=0; i<=n; ++i) if(i!=worst && f[i]>=f[second]) second=i;
+}
+
+template<typename T>
+T NM_simplex<T>::diameter() const
+{
+T d=0;
+for(int i=0; i<=n; ++i) if(i!=best)
+	{
+	NRVec<T> dx = v[i]-v[best];
+	d=std::max(d,dx.norm());
+	}
+return d;
+}
+
+//all vertices move towards the best one, as far as the evaluation budget allows
+template<typename T>
+void NM_simplex<T>::shrink(const T sigma, optevaluator<T> &e)
+{
+for(int i=0; i<=n; ++i) if(i!=best)
+	{
+	if(e.exhausted()) break;
+	NRVec<T> x = v[best]*(1-sigma);
+	x.axpy(sigma,v[i]);
+	v[i]=x;
+	f[i]=e(x);
+	}
+sumup();
+}
+
+
+template<typename T>
+optresult<T> NelderMead_Optimizer<T>::minimize(const OptProblem<T> *o, NRVec<T> &x, const hyperparameters<T> *par, hyperparameters<T> *achieved)
+{
+const char *name="Nelder-Mead";
+NelderMead_hyperparameters<T> hp;
+if(par) hp.setfrom(*par);
+if(hp.my_saddle_order) laerror("Nelder-Mead cannot search for saddle points");
+const int verbose=hp.my_verbose;
+const int n=x.size();
+T alpha=hp.my_alpha, gamma=hp.my_gamma, rho=hp.my_rho, sigma=hp.my_sigma;
+if(hp.my_adaptive) {alpha=1; gamma=1+(T)2/n; rho=0.75-(T)0.5/n; sigma=1-(T)1/n;}
+optevaluator<T> e(o,hp.my_maxfunc);
+optstate<T> st(name,hp,e);
+NM_simplex<T> s(x,hp.my_initstep,e);
+const T huge=std::numeric_limits<T>::max();
+
+for(;; ++st.iter)
+	{
+	s.order();
+	T spread=s.f[s.worst]-s.f[s.best];
+	st.step(spread, spread<hp.my_f_thr ? s.diameter() : huge); //diameter needed only if the spread is small
+	if(st.stop(s.f[s.best])) break;
+	if(st.iter%(n+1)==0) s.sumup(); //avoid accumulation of rounding errors
+
+	const char *op;
+	NRVec<T> c=s.centroid();
+	NRVec<T> xr=s.point(c,alpha);
+	T fr=e(xr);
+	if(fr<s.f[s.best]) //try expansion
+		{
+		if(e.exhausted()) {s.replace(s.worst,xr,fr); continue;}
+		NRVec<T> xe=s.point(c,alpha*gamma);
+		T fe=e(xe);
+		if(fe<fr) {s.replace(s.worst,xe,fe); op="expansion";}
+		else {s.replace(s.worst,xr,fr); op="reflection";}
+		}
+	else if(fr<s.f[s.second]) {s.replace(s.worst,xr,fr); op="reflection";}
+	else //contraction, outside if the reflected point is better than the worst one
+		{
+		bool outside= fr<s.f[s.worst];
+		if(e.exhausted()) {if(outside) s.replace(s.worst,xr,fr); continue;}
+		NRVec<T> xc=s.point(c, outside ? alpha*rho : -rho);
+		T fc=e(xc);
+		if(outside ? fc<=fr : fc<s.f[s.worst]) {s.replace(s.worst,xc,fc); op= outside ? "outside contraction" : "inside contraction";}
+		else {s.shrink(sigma,e); op="shrink";}
+		}
+	if(verbose>1) std::cout <<name<<" "<<op<<std::endl;
+	}
+
+s.order();
+x=s.v[s.best];
+st.step(s.f[s.worst]-s.f[s.best],s.diameter()); //exact values for achieved
+return st.finish(achieved);
+}
+
+
+
+template<typename T>
+optresult<T> Powell_Optimizer<T>::minimize(const OptProblem<T> *o, NRVec<T> &x, const hyperparameters<T> *par, hyperparameters<T> *achieved)
+{
+const char *name="Powell";
+Powell_hyperparameters<T> hp;
+if(par) hp.setfrom(*par);
+if(hp.my_saddle_order) laerror("Powell cannot search for saddle points");
+const int verbose=hp.my_verbose;
+const T abstol=0.01*hp.my_x_thr; //line minima have to be more precise than the requested accuracy in x
+const int n=x.size();
+optevaluator<T> e(o,hp.my_maxfunc);
+optstate<T> st(name,hp,e);
+T f=e(x);
+
+NRVec<NRVec<T> > u(n); //normalized directions
+for(int i=0; i<n; ++i) {u[i]=NRVec<T>((T)0,n); u[i][i]=1;}
+NRVec<T> tstep(hp.my_initstep,n); //trial steps of the line minimizations, from the last step along each direction
+
+for(;; ++st.iter)
+	{
+	if(st.stop(f)) break;
+	NRVec<T> x0(x);
+	T f0=f;
+	T delta=0; //largest decrease along a single direction
+	int ibig=0;
+	for(int i=0; i<n && !e.exhausted(); ++i)
+		{
+		T fprev=f;
+		T t=linemin_brent(e,x,f,u[i],tstep[i],hp.my_linmin_tol,abstol,hp.my_maxlinemin,verbose);
+		if(t!=0) tstep[i]=std::abs(t);
+		if(fprev-f>delta) {delta=fprev-f; ibig=i;}
+		}
+
+	//new direction x-x0 replaces the one of the largest decrease, if Powell's test indicates it is worth
+	NRVec<T> unew = x-x0;
+	T unorm=unew.norm();
+	if(unorm>0 && !e.exhausted())
+		{
+		NRVec<T> xe = x+unew; //extrapolated point
+		T fe=e(xe);
+		T a=f0-f-delta;
+		T b=f0-fe;
+		if(fe<f0 && 2*(f0-2*f+fe)*a*a < delta*b*b)
+			{
+			if(verbose>1) std::cout <<name<<": direction "<<ibig<<" replaced\n";
+			unew /= unorm;
+			u[ibig]=u[n-1]; tstep[ibig]=tstep[n-1];
+			u[n-1]=unew; tstep[n-1]=unorm;
+			if(!e.exhausted()) linemin_brent(e,x,f,unew,unorm,hp.my_linmin_tol,abstol,hp.my_maxlinemin,verbose);
+			}
+		}
+	NRVec<T> dx = x-x0;
+	st.step(f0-f,dx.norm());
+	}
+
+return st.finish(achieved);
+}
+
+
+
 //force instantiation
 template double linesearch_wolfe(optevaluator<double> &e, NRVec<double> &x, double &f, NRVec<double> &g, const NRVec<double> &p, const double alpha1, const linesearch_hyperparameters<double> &lp);
 template class optstate<double>;
@@ -449,6 +748,9 @@ template class testproblem<double>;
 template class BFGS_Optimizer<double>;
 template class LBFGS_Optimizer<double>;
 template class CG_Optimizer<double>;
+template class NelderMead_Optimizer<double>;
+template class Powell_Optimizer<double>;
+template double linemin_brent(optevaluator<double> &e, NRVec<double> &x, double &f, const NRVec<double> &u, const double t1, const double tol, const double abstol, const int maxiter, const int verbose);
 
 }//namespace
 

optimize.h

@@ -172,6 +172,13 @@ template<typename T>
 T linesearch_wolfe(optevaluator<T> &e, NRVec<T> &x, T &f, NRVec<T> &g, const NRVec<T> &p, const T alpha1, const linesearch_hyperparameters<T> &lp);
 
 
+//derivative-free line minimization of f(x + t*u): bracketing and Brent's method (Brent, Algorithms for Minimization without Derivatives, 1973, chap. 5)
+//on input x,f at the starting point, on output x,f at the minimum found; returns t, 0 if no decrease was found (x,f then unchanged)
+//t1 = initial trial step, tol and abstol = relative and absolute tolerance on t, maxiter = max. no. of function evaluations
+template<typename T>
+T linemin_brent(optevaluator<T> &e, NRVec<T> &x, T &f, const NRVec<T> &u, const T t1, const T tol, const T abstol, const int maxiter, const int verbose=0);
+
+
 //search direction of a line-search method, used by the common driver linesearch_minimize()
 template<typename T>
 class linesearch_direction {
@@ -261,6 +268,49 @@ virtual ~CG_Optimizer() {};
 };
 
 
+//Nelder-Mead downhill simplex (Lagarias et al., SIAM J. Optim. 9, 112 (1998)), derivative-free
+//convergence: f_thr is used for the spread of function values in the simplex, x_thr for its diameter, both have to be satisfied
+template<typename T>
+class NelderMead_hyperparameters : public hyperparameters<T> {
+public:
+T my_initstep; //edge length of the initial simplex x0 + initstep*e_i
+bool my_adaptive; //dimension-dependent coefficients (Gao & Han, Comput. Optim. Appl. 51, 259 (2012)), otherwise the following ones
+T my_alpha; //reflection
+T my_gamma; //expansion
+T my_rho; //contraction
+T my_sigma; //shrink
+NelderMead_hyperparameters() : my_initstep(0.1), my_adaptive(true), my_alpha(1), my_gamma(2), my_rho(0.5), my_sigma(0.5) {this->my_maxstep=5000; this->my_f_thr=1e-8; this->my_x_thr=1e-6;};
+virtual void setfrom(const hyperparameters<T> &p) {const NelderMead_hyperparameters *q=dynamic_cast<const NelderMead_hyperparameters *>(&p); if(q) *this= *q; else hyperparameters<T>::setfrom(p);}
+};
+
+template<typename T>
+class NelderMead_Optimizer : public Optimizer<T> {
+public:
+virtual optresult<T> minimize(const OptProblem<T> *o, NRVec<T> &x, const hyperparameters<T> *par=NULL, hyperparameters<T> *achieved=NULL);
+virtual ~NelderMead_Optimizer() {};
+};
+
+
+//Powell's conjugate direction method (Powell, Comput. J. 7, 155 (1964)), derivative-free
+//convergence: f_thr and x_thr are used for the change of f and x in one sweep over all directions, both have to be satisfied
+template<typename T>
+class Powell_hyperparameters : public hyperparameters<T> {
+public:
+T my_initstep; //initial trial step of the line minimizations
+T my_linmin_tol; //relative tolerance of the line minimizations, the absolute one is 0.01*x_thr
+int my_maxlinemin; //max. no. of function evaluations in one line minimization
+Powell_hyperparameters() : my_initstep(0.1), my_linmin_tol(1e-4), my_maxlinemin(50) {this->my_maxstep=1000; this->my_f_thr=1e-10; this->my_x_thr=1e-6;};
+virtual void setfrom(const hyperparameters<T> &p) {const Powell_hyperparameters *q=dynamic_cast<const Powell_hyperparameters *>(&p); if(q) *this= *q; else hyperparameters<T>::setfrom(p);}
+};
+
+template<typename T>
+class Powell_Optimizer : public Optimizer<T> {
+public:
+virtual optresult<T> minimize(const OptProblem<T> *o, NRVec<T> &x, const hyperparameters<T> *par=NULL, hyperparameters<T> *achieved=NULL);
+virtual ~Powell_Optimizer() {};
+};
+
+
 
 }//namespace
 #endif /* _OPTIMIZE_H_ */

t.cc

@@ -4845,22 +4845,28 @@ x.randomize(1.);
 testproblem<double> problem;
 //BFGS_hyperparameters<double> param; //tested
 //LBFGS_hyperparameters<double> param; //tested
-CG_hyperparameters<double> param;
+//CG_hyperparameters<double> param; //tested
+//NelderMead_hyperparameters<double> param; //tested
+Powell_hyperparameters<double> param;
 param.my_maxstep=100;
 param.my_f_thr=1e-6;
 param.my_x_thr=1e-3;
-param.my_g_thr=1e-3;
+//param.my_g_thr=1e-3; //gradient methods
 //param.my_update=QN_BFGS; //tested
 //param.my_update=QN_DFP; //tested
 //param.my_update=QN_SR1; //tested
 cout << "initial " <<x<<endl;
 //BFGS_Optimizer<double> o; //tested
 //LBFGS_Optimizer<double> o; //tested
-CG_Optimizer<double> o;
+//CG_Optimizer<double> o; //tested
-gradient_hyperparameters<double> achieved;
+//NelderMead_Optimizer<double> o; //tested
+Powell_Optimizer<double> o;
+//gradient_hyperparameters<double> achieved; //gradient methods
+hyperparameters<double> achieved;
 optresult<double> res=o.minimize(&problem,x,&param,&achieved);
 cout << "optimized " <<x<<endl;
-cout << "converged "<<res.converged<<" f= "<<res.f<<" iterations "<<achieved.my_maxstep<<" function evaluations "<<achieved.my_maxfunc<<" last df "<<achieved.my_f_thr<<" last dx "<<achieved.my_x_thr<<" |g| "<<achieved.my_g_thr<<endl;
+//cout << "converged "<<res.converged<<" f= "<<res.f<<" iterations "<<achieved.my_maxstep<<" function evaluations "<<achieved.my_maxfunc<<" last df "<<achieved.my_f_thr<<" last dx "<<achieved.my_x_thr<<" |g| "<<achieved.my_g_thr<<endl; //gradient methods
+cout << "converged "<<res.converged<<" f= "<<res.f<<" iterations "<<achieved.my_maxstep<<" function evaluations "<<achieved.my_maxfunc<<" last df "<<achieved.my_f_thr<<" last dx "<<achieved.my_x_thr<<endl;
 double r=problem.funcgrad(x,NULL);
 cout <<r<<endl;
 }