optimize: SPSA and Adam optimizers, generalized call_on_me() in vec.h

t.cc

@@ -4847,11 +4847,16 @@ testproblem<double> problem;
 //LBFGS_hyperparameters<double> param; //tested
 //CG_hyperparameters<double> param; //tested
 //NelderMead_hyperparameters<double> param; //tested
-Powell_hyperparameters<double> param;
+//Powell_hyperparameters<double> param; //tested
+//SGD_hyperparameters<double> param; //tested
+//Adam_hyperparameters<double> param; //tested
+SPSA_hyperparameters<double> param;
 param.my_maxstep=100;
 param.my_f_thr=1e-6;
 param.my_x_thr=1e-3;
-//param.my_g_thr=1e-3; //gradient methods
+param.my_g_thr=1e-3; //gradient methods
+param.my_lr=1e-2; //first-order methods
+param.my_maxstep=2000; //first-order methods
 //param.my_update=QN_BFGS; //tested
 //param.my_update=QN_DFP; //tested
 //param.my_update=QN_SR1; //tested
@@ -4860,13 +4865,16 @@ cout << "initial " <<x<<endl;
 //LBFGS_Optimizer<double> o; //tested
 //CG_Optimizer<double> o; //tested
 //NelderMead_Optimizer<double> o; //tested
-Powell_Optimizer<double> o;
-//gradient_hyperparameters<double> achieved; //gradient methods
-hyperparameters<double> achieved;
+//Powell_Optimizer<double> o; //tested
+//SGD_Optimizer<double> o; //tested
+//Adam_Optimizer<double> o; //tested
+SPSA_Optimizer<double> o;
+gradient_hyperparameters<double> achieved; //gradient methods
+//hyperparameters<double> achieved; //derivative-free methods
 optresult<double> res=o.minimize(&problem,x,&param,&achieved);
 cout << "optimized " <<x<<endl;
-//cout << "converged "<<res.converged<<" f= "<<res.f<<" iterations "<<achieved.my_maxstep<<" function evaluations "<<achieved.my_maxfunc<<" last df "<<achieved.my_f_thr<<" last dx "<<achieved.my_x_thr<<" |g| "<<achieved.my_g_thr<<endl; //gradient methods
-cout << "converged "<<res.converged<<" f= "<<res.f<<" iterations "<<achieved.my_maxstep<<" function evaluations "<<achieved.my_maxfunc<<" last df "<<achieved.my_f_thr<<" last dx "<<achieved.my_x_thr<<endl;
+cout << "converged "<<res.converged<<" f= "<<res.f<<" iterations "<<achieved.my_maxstep<<" function evaluations "<<achieved.my_maxfunc<<" last df "<<achieved.my_f_thr<<" last dx "<<achieved.my_x_thr<<" |g| "<<achieved.my_g_thr<<endl; //gradient methods
+//cout << "converged "<<res.converged<<" f= "<<res.f<<" iterations "<<achieved.my_maxstep<<" function evaluations "<<achieved.my_maxfunc<<" last df "<<achieved.my_f_thr<<" last dx "<<achieved.my_x_thr<<endl; //derivative-free methods
 double r=problem.funcgrad(x,NULL);
 cout <<r<<endl;
 }