references_old.bib

@InProceedings{atkeson1997,
	author =       "Atkeson, Christopher G. and Santamar\'{i}a, Juan Carlos",
	title =        "A Comparison of Direct and Model-Based Reinforcement Learning",
	booktitle =    "Proceedings of the 1997 {IEEE} International Conference on Robotics and Automation",
	year =         "1997",
	ISBN =         "0-7803-3612-7",
	volume =    "4",
	publisher = "{IEEE} Press",
	pages =     "3557--3564",
}

@incollection{bacciu2015probabilistic,
	title={Probabilistic Modeling in Machine Learning},
	author={Bacciu, Davide and Lisboa, Paulo JG and Sperduti, Alessandro and Villmann, Thomas},
	booktitle={Springer Handbook of Computational Intelligence},
	pages={545--575},
	year={2015},
	publisher={Springer}
}

@inproceedings{baker1992large,
	title={Large Vocabulary Recognition of Wall Street Journal Sentences at Dragon Systems},
	author={Baker, James and Baker, Janet and Bamberg, Paul and Bishop, Kathleen and Gillick, Larry and Helman, Vera and Huang, Zezhen and Ito, Yoshiko and Lowe, Stephen and Peskin, Barbara and others},
	booktitle={Proceedings of the workshop on Speech and Natural Language},
	pages={387--392},
	year={1992},
	organization={Association for Computational Linguistics}
}

@BOOK{barberBRML2012,
	author = {Barber, D.},
	title= {{Bayesian Reasoning and Machine Learning}},
	publisher = {{Cambridge University Press}},
	year = 2012}

@article{barto1995learning,
	title={Learning to act using real-time dynamic programming},
	author={Barto, Andrew G and Bradtke, Steven J and Singh, Satinder P},
	journal={Artificial intelligence},
	volume={72},
	number={1-2},
	pages={81--138},
	year={1995},
	publisher={Elsevier}
}

@inproceedings{bergstra2011algorithms,
	title={Algorithms for hyper-parameter optimization},
	author={Bergstra, James S and Bardenet, R{\'e}mi and Bengio, Yoshua and K{\'e}gl, Bal{\'a}zs},
	booktitle={Advances in Neural Information Processing Systems},
	pages={2546--2554},
	year={2011}
}

@inproceedings{bhatia2010sampling,
	title={Sampling-based motion planning with temporal goals},
	author={Bhatia, Amit and Kavraki, Lydia E and Vardi, Moshe Y},
	booktitle={Robotics and Automation (ICRA), 2010 IEEE International Conference on},
	pages={2689--2696},
	year={2010},
	organization={IEEE}
}

@article{bilmes1998gentle,
	title={A gentle tutorial of the EM algorithm and its application to parameter estimation for Gaussian mixture and hidden Markov models},
	author={Bilmes, Jeff A and others},
	journal={International Computer Science Institute},
	volume={4},
	number={510},
	pages={126},
	year={1998}
}

@article{Boutilier1999,
	abstract = {Planning under uncertainty is a central problem in the study of automated sequential decision making, and has been addressed by researchers in many different fields, including AI planning, decision analysis, operations research, control theory and economics. While the assumptions and perspectives adopted in these areas often differ in substantial ways, many planning problems of interest to researchers in these fields can be modeled as Markov decision processes (MDPs) and analyzed using the techniques of decision theory. This paper presents an overview and synthesis of MDP-related methods, showing how they provide a unifying framework for modeling many classes of planning problems studied in AI. It also describes structural properties of MDPs that, when exhibited by particular classes of problems, can be exploited in the construction of optimal or approximately optimal policies or plans. Planning problems commonly possess structure in the reward and value functions used to describe performance criteria, in the functions used to describe state transitions and observations, and in the relationships among features used to describe states, actions, rewards, and observations.},
	archivePrefix = {arXiv},
	arxivId = {1105.5460},
	author = {Boutilier, Craig and Dean, Thomas and Hanks, Steve},
	doi = {doi:10.1613/jair.575},
	eprint = {1105.5460},
	isbn = {90-5199-237-8},
	issn = {10769757},
	journal = {Journal of Artificial Intelligence Research},
	pages = {1--94},
	title = {{Decision-Theoretic Planning: Structural Assumptions and Computational Leverage}},
	volume = {11},
	year = {1999}
}

@article{Brafman2002,
	abstract = {Presents R-Max: a simple model-based RL algorithm, achieving near-optimal average reward in polynomial time. Initialised optimistically, to think all unknown transitions return maximal reward and move system to a default state. Handles explore/exploit trade-off implicitly. Works on stochastic games, being a transition system of two-player matrix games (as a general case of MDPs).},
	author = {Brafman, Ronen I and Tennenholtz, Moshe},
	doi = {10.1162/153244303765208377},
	file = {:C$\backslash$:/Users/Rob/TU Delft/Thesis/Papers/MDP Model Learning/R-MAX - A General Poly-time Algorithm for Near-Optimal RL.pdf:pdf},
	isbn = {1532-4435},
	issn = {15324435},
	journal = {Journal of Machine Learning Research},
	keywords = {decision processes,learning games,markov,provably efficient learning,reinforcement learning,stochastic games},
	mendeley-groups = {MDP Model Learning},
	pages = {213--231},
	title = {{R-MAX -- A General Polynomial Time Algorithm for Near-Optimal Reinforcement Learning}},
	volume = {3},
	year = {2002}
}

@article{Brochu2010,
	abstract = {We present a tutorial on Bayesian optimization, a method of finding the maximum of expensive cost functions. Bayesian optimization employs the Bayesian technique of setting a prior over the objective function and combining it with evidence to get a posterior function. This permits a utility-based selection of the next observation to make on the objective function, which must take into account both exploration (sampling from areas of high uncertainty) and exploitation (sampling areas likely to offer improvement over the current best observation). We also present two detailed extensions of Bayesian optimization, with experiments---active user modelling with preferences, and hierarchical reinforcement learning---and a discussion of the pros and cons of Bayesian optimization based on our experiences.},
	archivePrefix = {arXiv},
	arxivId = {1012.2599},
	author = {Brochu, E and Cora, V M and {De Freitas}, N},
	doi = {1012.2599},
	eprint = {1012.2599},
	file = {:C$\backslash$:/Users/Rob/TU Delft/Thesis/Papers/Bayesian Optimization/Tutorial on Bayesian Optimization.pdf:pdf},
	journal = {ArXiv},
	mendeley-groups = {Bayesian Optimization},
	pages = {49},
	title = {{A tutorial on Bayesian optimization of expensive cost functions, with application to active user modeling and hierarchical reinforcement learning}},
	year = {2010}
}

@article{brochu2010portfolio,
	title={Portfolio allocation for Bayesian optimization},
	author={Brochu, Eric and Hoffman, Matthew W and de Freitas, Nando},
	journal={arXiv preprint arXiv:1009.5419},
	year={2010}
}

@inproceedings{buffet2005robust,
	title={Robust planning with (L) RTDP},
	author={Buffet, Olivier and Aberdeen, Douglas and others},
	booktitle={INTERNATIONAL JOINT CONFERENCE ON ARTIFICIAL INTELLIGENCE},
	volume={19},
	pages={1214},
	year={2005},
	organization={LAWRENCE ERLBAUM ASSOCIATES LTD}
}

@article{caelli2001shape,
	title={Shape tracking and production using hidden Markov models},
	author={Caelli, Terry and McCabe, Andrew and Briscoe, Garry},
	journal={International Journal of Pattern Recognition and Artificial Intelligence},
	volume={15},
	number={01},
	pages={197--221},
	year={2001},
	publisher={World Scientific}
}

@article{calinon2007learning,
	title={On learning, representing, and generalizing a task in a humanoid robot},
	author={Calinon, Sylvain and Guenter, Florent and Billard, Aude},
	journal={IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics)},
	volume={37},
	number={2},
	pages={286--298},
	year={2007},
	publisher={IEEE}
}

@article{chamie2015finite,
	title={Finite-Horizon Markov Decision Processes with State Constraints},
	author={Chamie, Mahmoud El and Acikmese, Behcet},
	journal={arXiv preprint arXiv:1507.01585},
	year={2015}
}

@article{cochran2001generic,
	title={Generic Markov models for availability estimation and failure characterization in petroleum refineries},
	author={Cochran, Jeffery K and Murugan, Arvindh and Krishnamurthy, Vijayalakshmi},
	journal={Computers \& Operations Research},
	volume={28},
	number={1},
	pages={1--12},
	year={2001},
	publisher={Elsevier}
}

@misc{cordwellpymdptoolbox,
	title={pymdptoolbox 4.0-b3},
	author={Cordwell, SAW},
	url={https://github.com/sawcordwell/pymdptoolbox}
}

@misc{nogueirabayesianoptimization,
	title={BayesianOptimization},
	author={Nogueira, Fernando M. F.},
	url={https://github.com/fmfn/BayesianOptimization}
}

@article{cronvall2009combining,
	title={Combining discrete-time Markov processes and probabilistic fracture mechanics in RI-ISI risk estimates},
	author={Cronvall, Otso and M{\"a}nnist{\"o}, Ilkka},
	journal={International Journal of Pressure Vessels and Piping},
	volume={86},
	number={11},
	pages={732--737},
	year={2009},
	publisher={Elsevier}
}

@article{dayan1992q,
	title={Q-learning},
	author={Dayan, Peter and Watkins, CJCH},
	journal={Machine learning},
	volume={8},
	number={3},
	pages={279--292},
	year={1992}
}

@article{Degris2010,
	author = {Degris, Thomas and Sigaud, Oliver},
	file = {:C$\backslash$:/Users/Rob/TU Delft/Thesis/Papers/Factored Markov Decision Processes.pdf:pdf},
	journal = {Markov Decision Processes in Artificial Intelligence},
	pages = {99--126},
	title = {{Factored Markov Decision Processes}},
	year = {2010}
}

@inproceedings{deisenroth2011pilco,
	title={PILCO: A model-based and data-efficient approach to policy search},
	author={Deisenroth, Marc and Rasmussen, Carl E},
	booktitle={Proceedings of the 28th International Conference on machine learning (ICML-11)},
	pages={465--472},
	year={2011}
}

@article{delgado2011efficient,
	title={Efficient solutions to factored MDPs with imprecise transition probabilities},
	author={Delgado, Karina Valdivia and Sanner, Scott and De Barros, Leliane Nunes},
	journal={Artificial Intelligence},
	volume={175},
	number={9-10},
	pages={1498--1527},
	year={2011},
	publisher={Elsevier}
}

@article{delgado2016real,
	title={Real-time dynamic programming for Markov decision processes with imprecise probabilities},
	author={Delgado, Karina V and de Barros, Leliane N and Dias, Daniel B and Sanner, Scott},
	journal={Artificial Intelligence},
	volume={230},
	pages={192--223},
	year={2016},
	publisher={Elsevier}
}

@article{el2008optimal,
	title={Optimal design of a cogeneration plant for power and desalination taking equipment reliability into consideration},
	author={El-Nashar, Ali M},
	journal={Desalination},
	volume={229},
	number={1},
	pages={21--32},
	year={2008},
	publisher={Elsevier}
}

@inproceedings{epshteyn2008active,
	title={Active reinforcement learning},
	author={Epshteyn, Arkady and Vogel, Adam and DeJong, Gerald},
	booktitle={Proceedings of the 25th international conference on Machine learning},
	pages={296--303},
	year={2008},
	organization={ACM}
}


@article{gales2008application,
	title={The application of hidden Markov models in speech recognition},
	author={Gales, Mark and Young, Steve},
	journal={Foundations and trends in signal processing},
	volume={1},
	number={3},
	pages={195--304},
	year={2008},
	publisher={Now Publishers Inc.}
}

@article{Ghahramani2000,
	abstract = {We introduce a new statistical model for time series that iteratively segments data into regimes with approximately linear dynamics and learnsthe parameters of each of these linear regimes. This model combines and generalizes two of the most widely used stochastic time-series models -- hidden Markov models and linear dynamical systems -- and is closely related to models that are widely used in the control and econometrics literatures. It can also be derived by extending the mixture of experts neural network (Jacobs, Jordan, Nowlan, {\&} Hinton, 1991) to its fully dynamical version, in which both expert and gating networks are recurrent. Inferring the posterior probabilities of the hidden states of this model is computationally intractable, and therefore the exact expectation maximization (EM) algorithm cannot be applied. However, we present a variational approximation that maximizes a lower bound on the log-likelihood and makes use of both the forward and backward recursions for hidden Markov models and the Kalman filter recursions for linear dynamical systems. We tested the algorithm on artificial data sets and a natural data set of respiration force from a patient with sleep apnea. The results suggest that variational approximations are a viable method for inference and learning in switching state-space models.},
	author = {Ghahramani, Z and Hinton, Geoffrey E.},
	doi = {10.1162/089976600300015619},
	file = {:C$\backslash$:/Users/Rob/TU Delft/Thesis/Papers/Variational Learning for Switching State-Space Models.pdf:pdf},
	isbn = {0899-7667 (Print)$\backslash$r0899-7667 (Linking)},
	issn = {0899-7667},
	journal = {Neural computation},
	number = {4},
	pages = {831--864},
	pmid = {10770834},
	title = {{Variational learning for switching state-space models.}},
	volume = {12},
	year = {2000}
}

@article{Ghahramani2015,
	abstract = {How can a machine learn from experience? Probabilistic modelling provides a framework for understanding what learning is, and has therefore emerged as one of the principal theoretical and practical approaches for designing machines that learn from data acquired through experience. The probabilistic framework, which describes how to represent and manipulate uncertainty about models and predictions, has a central role in scientific data analysis, machine learning, robotics, cognitive science and artificial intelligence. This Review provides an introduction to this framework, and discusses some of the state-of-the-art advances in the field, namely, probabilistic programming, Bayesian optimization, data compression and automatic model discovery.},
	author = {Ghahramani, Zoubin},
	doi = {10.1038/nature14541},
	file = {:C$\backslash$:/Users/Rob/TU Delft/Thesis/Papers/Bayesian Optimization/Probabilistic Machine Learning and Artificial Intelligence.pdf:pdf},
	isbn = {0028-0836},
	issn = {0028-0836},
	journal = {Nature},
	mendeley-groups = {Bayesian Optimization},
	number = {7553},
	pages = {452--459},
	pmid = {26017444},
	title = {{Probabilistic machine learning and artificial intelligence}},
	url = {http://dx.doi.org/10.1038/nature14541},
	volume = {521},
	year = {2015}
}

@article{ghavamzadeh2015bayesian,
	title={Bayesian reinforcement learning: A survey},
	author={Ghavamzadeh, Mohammad and Mannor, Shie and Pineau, Joelle and Tamar, Aviv and others},
	journal={Foundations and Trends{\textregistered} in Machine Learning},
	volume={8},
	number={5-6},
	pages={359--483},
	year={2015},
	publisher={Now Publishers, Inc.}
}

@article{givan2000bounded,
	title={Bounded-parameter Markov decision processes},
	author={Givan, Robert and Leach, Sonia and Dean, Thomas},
	journal={Artificial Intelligence},
	volume={122},
	number={1-2},
	pages={71--109},
	year={2000},
	publisher={Elsevier}
}

@article{grimshaw2011markov,
	title={Markov chain models for delinquency: Transition matrix estimation and forecasting},
	author={Grimshaw, Scott D and Alexander, William P},
	journal={Applied Stochastic Models in Business and Industry},
	volume={27},
	number={3},
	pages={267--279},
	year={2011},
	publisher={Wiley Online Library}
}

@inproceedings{gross2008shopbot,
	title={Shopbot: Progress in developing an interactive mobile shopping assistant for everyday use},
	author={Gross, H-M and Boehme, H-J and Schr{\"o}ter, Christof and M{\"u}ller, Steffen and K{\"o}nig, Alexander and Martin, Ch and Merten, Matthias and Bley, Andreas},
	booktitle={Systems, Man and Cybernetics, 2008. SMC 2008. IEEE International Conference on},
	pages={3471--3478},
	year={2008},
	organization={IEEE}
}

@inproceedings{guez2012efficient,
	title={Efficient Bayes-adaptive reinforcement learning using sample-based search},
	author={Guez, Arthur and Silver, David and Dayan, Peter},
	booktitle={Advances in Neural Information Processing Systems},
	pages={1025--1033},
	year={2012}
}

@article{GuillotS17,
	author    = {Matthieu Guillot and
	Gautier Stauffer},
	title     = {The Stochastic Shortest Path Problem : {A} polyhedral combinatorics
	perspective},
	journal   = {CoRR},
	volume    = {abs/1702.03186},
	year      = {2017},
	url       = {http://arxiv.org/abs/1702.03186},
	timestamp = {Wed, 07 Jun 2017 14:40:44 +0200},
	biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/GuillotS17},
	bibsource = {dblp computer science bibliography, http://dblp.org}
}

@article{hawes2016strands,
	title={The STRANDS Project: Long-Term Autonomy in Everyday Environments},
	author={Hawes, Nick and Burbridge, Chris and Jovan, Ferdian and Kunze, Lars and Lacerda, Bruno and Mudrov{\'a}, Lenka and Young, Jay and Wyatt, Jeremy and Hebesberger, Denise and K{\"o}rtner, Tobias and others},
	journal={arXiv preprint arXiv:1604.04384},
	year={2016}
}

@inproceedings{hester2010generalized,
	title={Generalized model learning for reinforcement learning on a humanoid robot},
	author={Hester, Todd and Quinlan, Michael and Stone, Peter},
	booktitle={Robotics and Automation (ICRA), 2010 IEEE International Conference on},
	pages={2369--2374},
	year={2010},
	organization={IEEE}
}

@inproceedings{hester2012rtmba,
	title={RTMBA: A real-time model-based reinforcement learning architecture for robot control},
	author={Hester, Todd and Quinlan, Michael and Stone, Peter},
	booktitle={Robotics and Automation (ICRA), 2012 IEEE International Conference on},
	pages={85--90},
	year={2012},
	organization={IEEE}
}

@inproceedings{hoffman2011portfolio,
	title={Portfolio Allocation for Bayesian Optimization.},
	author={Hoffman, Matthew D and Brochu, Eric and de Freitas, Nando},
	booktitle={UAI},
	pages={327--336},
	year={2011}
}

@article{howard1960dynamic,
	title={DYNAMIC PROGRAMMING AND MARKOV PROCESSES..},
	author={Howard, Ronald A},
	year={1960}
}

@article{kaelbling1996reinforcement,
	title={Reinforcement learning: A survey},
	author={Kaelbling, Leslie Pack and Littman, Michael L and Moore, Andrew W},
	journal={Journal of artificial intelligence research},
	volume={4},
	pages={237--285},
	year={1996}
}

@inproceedings{kawaguchi2015bayesian,
	title={Bayesian optimization with exponential convergence},
	author={Kawaguchi, Kenji and Kaelbling, Leslie Pack and Lozano-P{\'e}rez, Tom{\'a}s},
	booktitle={Advances in Neural Information Processing Systems},
	pages={2809--2817},
	year={2015}
}

@article{kearns2002near,
	title={Near-optimal reinforcement learning in polynomial time},
	author={Kearns, Michael and Singh, Satinder},
	journal={Machine Learning},
	volume={49},
	number={2-3},
	pages={209--232},
	year={2002},
	publisher={Springer}
}

@inproceedings{koenig1996unsupervised,
	title={Unsupervised learning of probabilistic models for robot navigation},
	author={Koenig, Sven and Simmons, Reid G},
	booktitle={Robotics and Automation, 1996. Proceedings., 1996 IEEE International Conference on},
	volume={3},
	pages={2301--2308},
	year={1996},
	organization={IEEE}
}

@inproceedings{lacerda2015optimal,
	title={Optimal Policy Generation for Partially Satisfiable Co-Safe LTL Specifications.},
	author={Lacerda, Bruno and Parker, David and Hawes, Nick},
	booktitle={IJCAI},
	pages={1587--1593},
	year={2015}
}

@article{lee1968maximum,
	title={Maximum likelihood and Bayesian estimation of transition probabilities},
	author={Lee, T Cv and Judge, GG and Zellner, Arnold},
	journal={Journal of the American Statistical Association},
	volume={63},
	number={324},
	pages={1162--1179},
	year={1968},
	publisher={Taylor \& Francis Group}
}

@inproceedings{littman1995complexity,
	title={On the complexity of solving Markov decision problems},
	author={Littman, Michael L and Dean, Thomas L and Kaelbling, Leslie Pack},
	booktitle={Proceedings of the Eleventh conference on Uncertainty in artificial intelligence},
	pages={394--402},
	year={1995},
	organization={Morgan Kaufmann Publishers Inc.}
}

@book{lizotte2008practical,
	title={Practical bayesian optimization},
	author={Lizotte, Daniel James},
	year={2008},
	publisher={University of Alberta}
}

@inproceedings{martinez2007active,
	title={Active Policy Learning for Robot Planning and Exploration under Uncertainty.},
	author={Martinez-Cantin, Ruben and de Freitas, Nando and Doucet, Arnaud and Castellanos, Jos{\'e} A},
	booktitle={Robotics: Science and Systems},
	pages={321--328},
	year={2007}
}

@article{MartinezCantin2009,
	abstract = {We address the problem of online path planning for optimal sensing with a mobile robot. The objective of the robot is to learn the most about its pose and the environment given time constraints. We use a POMDP with a utility function that depends on the belief state to model the finite horizon planning problem. We replan as the robot progresses throughout the environment. The POMDP is high-dimensional, continuous, non-differentiable, nonlinear, non-Gaussian and must be solved in real-time. Most existing techniques for stochastic planning and reinforcement learning are therefore inapplicable. To solve this extremely complex problem, we propose a Bayesian optimization method that dynamically trades off exploration (minimizing uncertainty in unknown parts of the policy space) and exploitation (capitalizing on the current best solution). We demonstrate our approach with a visually-guide mobile robot. The solution proposed here is also applicable to other closely-related domains, including active vision, sequential experimental design, dynamic sensing and calibration with mobile sensors.},
	archivePrefix = {arXiv},
	arxivId = {0712.3744},
	author = {Martinez-Cantin, Ruben and {De Freitas}, Nando and Brochu, Eric and Castellanos, Jos{\'{e}} and Doucet, Arnaud},
	doi = {10.1007/s10514-009-9130-2},
	eprint = {0712.3744},
	file = {:C$\backslash$:/Users/Rob/TU Delft/Thesis/Papers/Bayesian Optimization/A Bayesian exploration-exploitation approach for optimal online sensing and planning with a visually guided mobile robot.pdf:pdf},
	isbn = {0929-5593},
	issn = {09295593},
	journal = {Autonomous Robots},
	keywords = {Active SLAM,Active learning,Active vision,Attention and gaze planning,Bayesian optimization,Dynamic sensor networks,Model predictive control,Online path planning,Policy search,Reinforcement learning,Sequential experimental design},
	mendeley-groups = {Bayesian Optimization},
	number = {2},
	pages = {93--103},
	title = {{A Bayesian exploration-exploitation approach for optimal online sensing and planning with a visually guided mobile robot}},
	volume = {27},
	year = {2009}
}

@article{Melo20111757,
	title = "Decentralized \{MDPs\} with sparse interactions ",
	journal = "Artificial Intelligence ",
	volume = "175",
	number = "11",
	pages = "1757 - 1789",
	year = "2011",
	note = "",
	issn = "0004-3702",
	doi = "http://dx.doi.org/10.1016/j.artint.2011.05.001",
	url = "http://www.sciencedirect.com/science/article/pii/S0004370211000634",
	author = "Francisco S. Melo and Manuela Veloso",
	keywords = "Multiagent coordination",
	keywords = "Sparse interaction",
	keywords = "Decentralized Markov decision processes "
}

@misc{Metralabs,  url={http://www.metralabs.com/en/shopping-rfid-robot}, journal={MetraLabs}, publisher={MetraLabs}, author={MetraLabs}, year={2017}, month={Feb}}

@article{Minka1999,
	author = {Minka, T},
	file = {:C$\backslash$:/Users/Rob/TU Delft/Thesis/Papers/From Hidden Markov Models to Linear Dynamical Systems.pdf:pdf},
	journal = {Technical report, MIT},
	pmid = {1000198457},
	title = {{From Hidden Markov Models to Linear Dynamical Systems}},
	url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.51.1207{\&}rep=rep1{\&}type=pdf},
	year = {1999}
}

@article{minka2003bayesian,
	title={Bayesian inference, entropy, and the multinomial distribution},
	author={Minka, Tom},
	journal={Online tutorial},
	year={2003}
}

@article{Moldovan2012,
	abstract = {In environments with uncertain dynamics exploration is necessary to learn how to perform well. Existing reinforcement learning algorithms provide strong exploration guarantees, but they tend to rely on an ergodicity assumption. The essence of ergodicity is that any state is eventually reachable from any other state by following a suitable policy. This assumption allows for exploration algorithms that operate by simply favoring states that have rarely been visited before. For most physical systems this assumption is impractical as the systems would break before any reasonable exploration has taken place, i.e., most physical systems don't satisfy the ergodicity assumption. In this paper we address the need for safe exploration methods in Markov decision processes. We first propose a general formulation of safety through ergodicity. We show that imposing safety by restricting attention to the resulting set of guaranteed safe policies is NP-hard. We then present an efficient algorithm for guaranteed safe, but potentially suboptimal, exploration. At the core is an optimization formulation in which the constraints restrict attention to a subset of the guaranteed safe policies and the objective favors exploration policies. Our framework is compatible with the majority of previously proposed exploration methods, which rely on an exploration bonus. Our experiments, which include a Martian terrain exploration problem, show that our method is able to explore better than classical exploration methods.},
	archivePrefix = {arXiv},
	arxivId = {1205.4810},
	author = {Moldovan, Teodor Mihai and Abbeel, Pieter},
	eprint = {1205.4810},
	file = {:C$\backslash$:/Users/Rob/TU Delft/Thesis/Papers/Bayesian Optimization/Safe Exploration (SafeMDP) in Finite Markov Decision Processes with Gaussin Processes.pdf:pdf},
	isbn = {978-1-4503-1285-1},
	journal = {Proceedings of the 29th International Conference on Machine Learning},
	keywords = {ICML,exploration,machine learning,safe,safety},
	mendeley-groups = {Bayesian Optimization},
	number = {Nips},
	title = {{Safe Exploration in Markov Decision Processes}},
	url = {http://arxiv.org/abs/1205.4810},
	year = {2012}
}

@inproceedings{morse_simpar_2012,
	author    = {Gilberto Echeverria and
	Séverin Lemaignan and
	Arnaud Degroote and
	Simon Lacroix and
	Michael Karg and
	Pierrick Koch and
	Charles Lesire and
	Serge Stinckwich},
	title     = {Simulating Complex Robotic Scenarios with MORSE},
	booktitle = {SIMPAR},
	year      = {2012},
	pages     = {197-208},
	url       = {http://morse.openrobots.org}
}

@inproceedings{nikovski1999learning,
	title={Learning discrete Bayesian models for autonomous agent navigation},
	author={Nikovski, Daniel and Nourbakhsh, Illah},
	booktitle={Computational Intelligence in Robotics and Automation, 1999. CIRA'99. Proceedings. 1999 IEEE International Symposium on},
	pages={137--143},
	year={1999},
	organization={IEEE}
}

@inproceedings{nikovski2000learning,
	title={Learning Probabilistic Models for Decision-Theoretic Navigation of Mobile Robots.},
	author={Nikovski, Daniel and Nourbakhsh, Illah R},
	booktitle={ICML},
	pages={671--678},
	year={2000},
	organization={Citeseer}
}

@phdthesis{nikovski2002state,
	title={State-aggregation algorithms for learning probabilistic models for robot control},
	author={Nikovski, Daniel and Nourbakhsh, Illah},
	year={2002},
	school={Carnegie Mellon University, The Robotics Institute}
}


@incollection{parent1991stochastic,
	title={Stochastic modeling of a water resource system: analytical techniques versus synthetic approaches},
	author={Parent, E and Lebdi, F and Hurand, P},
	booktitle={Water resources engineering risk assessment},
	pages={415--434},
	year={1991},
	publisher={Springer}
}

@article{pasanisi2012estimating,
	title={Estimating discrete Markov models from various incomplete data schemes},
	author={Pasanisi, Alberto and Fu, Shuai and Bousquet, Nicolas},
	journal={Computational Statistics \& Data Analysis},
	volume={56},
	number={9},
	pages={2609--2625},
	year={2012},
	publisher={Elsevier}
}

@phdthesis{pazis2012non,
	title={Non-parametric approximate linear programming for MDPs},
	author={Pazis, Jason},
	year={2012},
	school={Citeseer}
}

@article{perchet2014gaussian,
	title={Gaussian process optimization with mutual information},
	author={Perchet, Vianney},
	year={2014}
}

@inproceedings{png2011bayesian,
	title={Bayesian reinforcement learning for POMDP-based dialogue systems},
	author={Png, ShaoWei and Pineau, Joelle},
	booktitle={Acoustics, Speech and Signal Processing (ICASSP), 2011 IEEE International Conference on},
	pages={2156--2159},
	year={2011},
	organization={IEEE}
}

@book{poole2010artificial,
	title={Artificial Intelligence: foundations of computational agents},
	author={Poole, David L and Mackworth, Alan K},
	year={2010},
	publisher={Cambridge University Press}
}

@Inbook{Poupart2010,
	author="Poupart, Pascal",
	editor="Sammut, Claude
	and Webb, Geoffrey I.",
	title="Bayesian Reinforcement Learning",
	bookTitle="Encyclopedia of Machine Learning",
	year="2010",
	publisher="Springer US",
	address="Boston, MA",
	pages="90--93",
	isbn="978-0-387-30164-8",
	doi="10.1007/978-0-387-30164-8_67",
	url="http://dx.doi.org/10.1007/978-0-387-30164-8_67"
}

@book{puterman2014markov,
	title={Markov decision processes: discrete stochastic dynamic programming},
	author={Puterman, Martin L},
	year={2014},
	publisher={John Wiley \& Sons}
}

@article{rabiner1989tutorial,
	title={A tutorial on hidden Markov models and selected applications in speech recognition},
	author={Rabiner, Lawrence R},
	journal={Proceedings of the IEEE},
	volume={77},
	number={2},
	pages={257--286},
	year={1989},
	publisher={IEEE}
}

@book{rasmussen2006gaussian,
	title={Gaussian processes for machine learning},
	author={Rasmussen, Carl Edward and Williams, Christopher KI},
	volume={1},
	year={2006},
	publisher={MIT press Cambridge}
}

@article{satia1973markovian,
	title={Markovian decision processes with uncertain transition probabilities},
	author={Satia, Jay K and Lave Jr, Roy E},
	journal={Operations Research},
	volume={21},
	number={3},
	pages={728--740},
	year={1973},
	publisher={INFORMS}
}

@incollection{schaefer2005modeling,
	title={Modeling medical treatment using Markov decision processes},
	author={Schaefer, Andrew J and Bailey, Matthew D and Shechter, Steven M and Roberts, Mark S},
	booktitle={Operations research and health care},
	pages={593--612},
	year={2005},
	publisher={Springer}
}

@article{shahriari2016taking,
	title={Taking the human out of the loop: A review of bayesian optimization},
	author={Shahriari, Bobak and Swersky, Kevin and Wang, Ziyu and Adams, Ryan P and de Freitas, Nando},
	journal={Proceedings of the IEEE},
	volume={104},
	number={1},
	pages={148--175},
	year={2016},
	publisher={IEEE}
}

@techreport{shatkay1997learning,
	title={Learning hidden Markov models with geometric information},
	author={Shatkay, Hagit and Kaelbling, Leslie Pack},
	year={1997},
	institution={Citeseer}
}

@inproceedings{silver2008sample,
	title={Sample-based learning and search with permanent and transient memories},
	author={Silver, David and Sutton, Richard S and M{\"u}ller, Martin},
	booktitle={Proceedings of the 25th international conference on Machine learning},
	pages={968--975},
	year={2008},
	organization={ACM}
}

@article{singh2000convergence,
	title={Convergence results for single-step on-policy reinforcement-learning algorithms},
	author={Singh, Satinder and Jaakkola, Tommi and Littman, Michael L and Szepesv{\'a}ri, Csaba},
	journal={Machine learning},
	volume={38},
	number={3},
	pages={287--308},
	year={2000},
	publisher={Springer}
}

@inproceedings{snoek2012practical,
	title={Practical bayesian optimization of machine learning algorithms},
	author={Snoek, Jasper and Larochelle, Hugo and Adams, Ryan P},
	booktitle={Advances in neural information processing systems},
	pages={2951--2959},
	year={2012}
}

@article{stolcke1994best,
	title={Best-first model merging for hidden Markov model induction},
	author={Stolcke, Andreas and Omohundro, Stephen M},
	journal={arXiv preprint cmp-lg/9405017},
	year={1994}
}

@book{sutton1998reinforcement,
	title={Reinforcement learning: An introduction},
	author={Sutton, Richard S and Barto, Andrew G},
	volume={1},
	number={1},
	year={1998},
	publisher={MIT press Cambridge}
}

@article{teodorescu2009maximum,
	title={Maximum Likelihood Estimation for Markov Chains},
	author={Teodorescu, Iuliana},
	journal={arXiv preprint arXiv:0905.4131},
	year={2009},
	publisher={Citeseer}
}

@inproceedings{turchetta2016safe,
	title={Safe exploration in finite Markov decision processes with Gaussian processes},
	author={Turchetta, Matteo and Berkenkamp, Felix and Krause, Andreas},
	booktitle={Advances in Neural Information Processing Systems},
	pages={4305--4313},
	year={2016}
}

@inproceedings{wang2016optimization,
	title={Optimization as estimation with Gaussian processes in bandit settings},
	author={Wang, Zi and Zhou, Bolei and Jegelka, Stefanie},
	booktitle={International Conf. on Artificial and Statistics (AISTATS)},
	year={2016}
}

@article{welch2003hidden,
	title={Hidden Markov models and the Baum-Welch algorithm},
	author={Welch, Lloyd R},
	journal={IEEE Information Theory Society Newsletter},
	volume={53},
	number={4},
	pages={10--13},
	year={2003}
}

@misc{wiering2004intelligent,
	title={Intelligent traffic light control},
	author={Wiering, MA and Veenen, J van and Vreeken, Jilles and Koopman, Arne},
	year={2004},
	publisher={Utrecht University: Information and Computing Sciences}
}

@article{wilson2014using,
	title={Using trajectory data to improve bayesian optimization for reinforcement learning.},
	author={Wilson, Aaron and Fern, Alan and Tadepalli, Prasad},
	journal={Journal of Machine Learning Research},
	volume={15},
	number={1},
	pages={253--282},
	year={2014}
}