bibliography.bib

Automatically generated by Mendeley Desktop 1.14
Any changes to this file will be lost if it is regenerated by Mendeley.

BibTeX export options can be customized via Preferences -> BibTeX in Mendeley Desktop

@book{Devroye1997,
abstract = {Pattern recognition presents one of the most significant challenges for scientists and engineers, and many different approaches have been proposed. The aim of this book is to provide a self-contained account of probabilistic analysis of these approaches. The book includes a discussion of distance measures, nonparametric methods based on kernels or nearest neighbors, Vapnik-Chervonenkis theory, epsilon entropy, parametric classification, error estimation, tree classifiers, and neural networks. Wherever possible, distribution-free properties and inequalities are derived. A substantial portion of the results or the analysis is new. Over 430 problems and exercises complement the material.},
author = {Devroye, Luc and Györfi, László and Lugosi, Gábor},
doi = {10.1007/978-1-4612-0711-5},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Devroye, Györfi, Lugosi - 1997 - A Probabilistic Theory of Pattern Recognition.pdf:pdf},
isbn = {978-0387946184},
pages = {661},
title = {{A Probabilistic Theory of Pattern Recognition}},
year = {1997}
}
@article{Leistner2010,
author = {Leistner, Christian and Saffari, Amir and Bischof, Horst},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Leistner, Saffari, Bischof - 2010 - MIForests Multiple-instance learning with randomized trees.pdf:pdf},
journal = {11th ECCV},
number = {825840},
pages = {29--42},
title = {{MIForests: Multiple-instance learning with randomized trees}},
year = {2010}
}
@article{Breiman2004a,
author = {Breiman, Leo},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Breiman - 2004 - Consistency for a simple model of random forests.pdf:pdf},
journal = {Technical Report 670},
title = {{Consistency for a simple model of random forests}},
url = {http://scholar.google.com/scholar?hl=en\&btnG=Search\&q=intitle:CONSISTENCY+FOR+A+SIMPLE+MODEL+OF+RANDOM+FORESTS\#0},
year = {2004}
}
@article{Lopez2014,
abstract = {A wide number of real word applications presents a class distribution where examples belonging to one class heavily outnumber the examples in the other class. This is an arduous situation where standard classification techniques usually decrease their performance, creating a handicap to correctly identify the minority class, which is precisely the case under consideration in these applications.In this work, we propose the usage of the Iterative Instance Adjustment for Imbalanced Domains (IPADE-ID) algorithm. It is an evolutionary framework, which uses an instance generation technique, designed to face the existing imbalance modifying the original training set. The method, iteratively learns the appropriate number of examples that represent the classes and their particular positioning. The learning process contains three key operations in its design: a customized initialization procedure, an evolutionary optimization of the positioning of the examples and a selection of the most representative examples for each class.An experimental analysis is carried out with a wide range of highly imbalanced datasets over the proposal and recognized solutions to the problem. The results obtained, which have been contrasted through non-parametric statistical tests, show that our proposal outperforms previously proposed methods. © 2013 Elsevier B.V.},
author = {L\'{o}pez, Victoria and Triguero, Isaac and Carmona, Crist\'{o}bal J. and Garc\'{\i}a, Salvador and Herrera, Francisco},
doi = {10.1016/j.neucom.2013.01.050},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/L\'{o}pez et al. - 2014 - Addressing imbalanced classification with instance generation techniques IPADE-ID.pdf:pdf},
issn = {09252312},
journal = {Neurocomputing},
keywords = {Decision tree,Differential evolution,Imbalanced datasets,Instance generation,Nearest neighbor},
pages = {15--28},
title = {{Addressing imbalanced classification with instance generation techniques: IPADE-ID}},
volume = {126},
year = {2014}
}
@article{Swersky2014,
abstract = {In machine learning, the term “training” is used to describe the procedure of fitting a model to data. In many popular models, this fitting procedure is framed as an optimization problem, in which a loss is minimized as a function of the parameters. In all but the simplest machine learning models, this minimization must be performed with an iterative algorithm such as stochastic gradient descent or the nonlinear conjugate gradient method. Another aspect of training involves fitting model “hyperparameters.” These are parameters that in some way govern the model space or fitting procedure; in both cases they are typically difficult to minimize directly in terms of the training loss and are usually evaluated in terms of generalization performance via held-out data. Hyperparameters are often regularization penalties such as ?p norms on model parameters, but can also capture model capacity as in the number of hidden units in a neural network. These hyperparameters help determine the appropriate bias-variance tradeoff for a given model family and data set. On the other hand, hyperparameters of the fitting procedure govern algorithmic aspects of training, such as the learning rate schedule of stochastic gradient descent, or the width of a Monte Carlo proposal distribution. The goal of fitting both kinds of hyperparameters is to identify a model and an optimization procedure in which successful minimization of training loss is likely to result in good generalization performance. When a held- out validation set is used to evaluate the quality of hyperparameters, the overall optimization proceeds as a double loop, where the outer loop sets the hyperparameters and the inner loop applies an iterative training procedure to fit the model to data. Often this outer hyperparameter optimization is performed by hand, which—even if rigorously systematized— can be a difficult and laborious process. Simple alternatives include the application of heuristics and intu- ition, grid search, which scales poorly with dimension, or random search [1], which is computationally expensive due to the need to train many models. In light of this, Bayesian optimization [2] has recently been proposed as an effective method for systematically and intelligently setting the hyperparameters of machine learning models [3, 4]. Using a principled characterization of model uncertainty, Bayesian optimization attempts to find the best hyperparameter settings with as few model evaluations as possible. One issue with previously proposed approaches to Bayesian optimization for machine learning is that a model must be fully trained before the quality of its hyperparameters can be assessed. Human experts, however, appear to be able to rapidly assess whether or not a model is likely to eventually be useful, even when the inner-loop training is only partially completed. When such an assessment can be made accurately, it is possible to explore the hyperparameter space more effectively by aborting model fits that are likely to be low quality. The goal of this paper is to take advantage of the partial information provided by iterative training procedures, within the Bayesian optimization framework for hyperparameter search.We propose a new technique that makes it possible to estimate when to pause the training of one model in favor of starting a new one with different hyperparameters, or resuming a partially-completed training procedure from an old model.We refer to our approach as freeze-thaw Bayesian optimization, as the algorithm maintains a set of “frozen” (partially completed but not being actively trained) models and uses an information-theoretic criterion to determine which ones to “thaw” and continue training. Our approach hinges on the assumption that, for many models, the training loss during the fitting proce- dure roughly follows an exponential decay towards an unknown final value.We build a Bayesian nonpara- metric prior around this assumption by developing a new kernel that is an infinite mixture of exponentially- decaying basis functions, with the goal of characterizing these training curves. Using this kernel with a novel and efficient temporal Gaussian process prior, we are able to forecast the final result of partially trained mod- els and use this during Bayesian optimization to determine the most promising action.We demonstrate that freeze-thaw Bayesian optimization can find good hyperparameter settings for many different models in con- siderably less time than ordinary Bayesian optimization.},
archivePrefix = {arXiv},
arxivId = {arXiv:1406.3896v1},
author = {Swersky, Kevin and Snoek, Jasper and Adams, Ryan P.},
eprint = {arXiv:1406.3896v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Swersky, Snoek, Adams - 2014 - Freeze-Thaw Bayesian Optimization.pdf:pdf},
journal = {arXiv preprint},
pages = {1--12},
title = {{Freeze-Thaw Bayesian Optimization}},
url = {http://arxiv.org/abs/1406.3896},
year = {2014}
}
@article{Paleologo2010,
abstract = {The logistic regression framework has been for long time the most used statistical method when assessing customer credit risk. Recently, a more pragmatic approach has been adopted, where the first issue is credit risk prediction, instead of explanation. In this context, several classification techniques have been shown to perform well on credit scoring, such as support vector machines among others. While the investigation of better classifiers is an important research topic, the specific methodology chosen in real world applications has to deal with the challenges arising from the real world data collected in the industry. Such data are often highly unbalanced, part of the information can be missing and some common hypotheses, such as the i.i.d. one, can be violated. In this paper we present a case study based on a sample of IBM Italian customers, which presents all the challenges mentioned above. The main objective is to build and validate robust models, able to handle missing information, class unbalancedness and non-iid data points. We define a missing data imputation method and propose the use of an ensemble classification technique, subagging, particularly suitable for highly unbalanced data, such as credit scoring data. Both the imputation and subagging steps are embedded in a customized cross-validation loop, which handles dependencies between different credit requests. The methodology has been applied using several classifiers (kernel support vector machines, nearest neighbors, decision trees, Adaboost) and their subagged versions. The use of subagging improves the performance of the base classifier and we will show that subagging decision trees achieve better performance, still keeping the model simple and reasonably interpretable. © 2009 Elsevier B.V. All rights reserved.},
author = {Paleologo, Giuseppe and Elisseeff, Andr\'{e} and Antonini, Gianluca},
doi = {10.1016/j.ejor.2009.03.008},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Paleologo, Elisseeff, Antonini - 2010 - Subagging for credit scoring models.pdf:pdf},
issn = {03772217},
journal = {European Journal of Operational Research},
keywords = {Classification,Credit scoring,Decision Support Systems,Risk analysis},
number = {2},
pages = {490--499},
publisher = {Elsevier B.V.},
title = {{Subagging for credit scoring models}},
url = {http://dx.doi.org/10.1016/j.ejor.2009.03.008},
volume = {201},
year = {2010}
}
@article{Wright,
author = {Wright, Jonathan H},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wright - Unknown - Bayesian Model Averaging and Exchange Rate Forecasts Jonathan H. Wright.pdf:pdf},
keywords = {ben bernanke,board of governors of,bootstrap,c32,c53,david bowman,dc 20551,exchange rates,f31,forecasting,i am grateful to,international finance division,jel classification,jon faust,matt pritsker and pietro,model uncertainty,shrinkage,the federal reserve system,washington},
number = {1983},
title = {{Bayesian Model Averaging and Exchange Rate Forecasts Jonathan H. Wright *}}
}
@article{Ruiz2008,
abstract = {The aim of this study is to compare two supervised classification methods on a crucial meteorological problem. The data consist of satellite measurements of cloud systems which are to be classified either in convective or non convective systems. Convective cloud systems correspond to lightning and detecting such systems is of main importance for thunderstorm monitoring and warning. Because the problem is highly unbalanced, we consider specific performance criteria and different strategies. This case study can be used in an advanced course of data mining in order to illustrate the use of logistic regression and random forest on a real data set with unbalanced classes.},
archivePrefix = {arXiv},
arxivId = {0804.0650},
author = {Ruiz, Anne and Villa, Nathalie},
eprint = {0804.0650},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ruiz, Villa - 2008 - Storms prediction Logistic regression vs random forest for unbalanced data.pdf:pdf},
pages = {1--11},
title = {{Storms prediction : Logistic regression vs random forest for unbalanced data}},
url = {http://arxiv.org/abs/0804.0650},
year = {2008}
}
@article{Wilson1972,
abstract = {The convergence properties of a nearest neighbor rule that uses an editing procedure to reduce the number of preclassified samples and to improve the performance of the rule are developed. Editing of the preclassified samples using the three-nearest neighbor rule followed by classification using the single-nearest neighbor rule with the remaining preclassified samples appears to produce a decision procedure whose risk approaches the Bayes' risk quite closely in many problems with only a few preclassified samples. The asymptotic risk of the nearest neighbor rules and the nearest neighbor rules using edited preclassified samples is calculated for several problems.},
author = {Wilson, Dennis L.},
doi = {10.1109/TSMC.1972.4309137},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wilson - 1972 - Asymptotic Properties of Nearest Neighbor Rules Using Edited Data.pdf:pdf},
isbn = {0018-9472},
issn = {0018-9472},
journal = {IEEE Transactions on Systems, Man, and Cybernetics},
number = {3},
pages = {408--421},
title = {{Asymptotic Properties of Nearest Neighbor Rules Using Edited Data}},
volume = {2},
year = {1972}
}
@article{Achlioptas2001,
author = {Achlioptas, Dimitris},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Achlioptas - 2001 - Database-friendly Random Projections.pdf:pdf},
isbn = {1581133618},
pages = {274--281},
title = {{Database-friendly Random Projections}},
year = {2001}
}
@article{Fawcett2008,
abstract = {Rules are commonly used for classification because they are modular, intelligible and easy to learn. Existing work in classification rule learning assumes the goal is to produce categorical classifications to maximize classification accuracy. Recent work in machine learning has pointed out the limitations of classification accuracy: when class distributions are skewed, or error costs are unequal, an accuracy maximizing classifier can perform poorly. This paper presents a method for learning rules directly from ROC space when the goal is to maximize the area under the ROC curve (AUC). Basic principles from rule learning and computational geometry are used to focus search for promising rule combinations. The result is a system that can learn intelligible rulelists with good ROC performance.},
author = {Fawcett, Tom},
doi = {10.1007/s10618-008-0089-y},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Fawcett - 2008 - PRIE A system for generating rulelists to maximize ROC performance.pdf:pdf},
issn = {13845810},
journal = {Data Mining and Knowledge Discovery},
keywords = {Classification,Cost-sensitive learning,ROC analysis,Rule learning},
number = {2},
pages = {207--224},
title = {{PRIE: A system for generating rulelists to maximize ROC performance}},
volume = {17},
year = {2008}
}
@article{Hoeting1999b,
abstract = {Standard statistical practice ignores model uncertainty. Data analysts typically select a model from some class of models and then proceed as if the selected model had generated the data. This approach ignores the uncertainty in model selection, leading to over-confident inferences and decisions that are more risky than one thinks they are. Bayesian model averaging (BMA) provides a coherent mechanism for accounting for this model uncertainty. Several methods for implementing BMA have recently emerged. We discuss these methods and present a number of examples. In these examples, BMA provides improved out-of- sample predictive performance. We also provide a catalogue of currently available BMA software.},
author = {Hoeting, Jennifer a and Madigan, David and Raftery, Adrian E and Volinsky, C T},
doi = {10.2307/2676803},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hoeting et al. - 1999 - Bayesian model averaging a tutorial.pdf:pdf},
issn = {08834237},
journal = {Statistical Science},
keywords = {Bayesian graphical models,Bayesian model averaging,Markov chain Monte Carlo.,learning,model uncertainty},
number = {4},
pages = {382--417},
title = {{Bayesian model averaging: a tutorial}},
url = {http://www.jstor.org/stable/2676803},
volume = {14},
year = {1999}
}
@book{Swersky,
abstract = {Bayesian optimization has recently been proposed as a framework for automati- cally tuning the hyperparameters of machine learning models and has been shown to yield state-of-the-art performance with impressive ease and efficiency. In this paper, we explore whether it is possible to transfer the knowledge gained from previous optimizations to new tasks in order to find optimal hyperparameter set- tings more efficiently. Our approach is based on extending multi-task Gaussian processes to the framework of Bayesian optimization. We show that this method significantly speeds up the optimization process when compared to the standard single-task approach. We further propose a straightforward extension of our al- gorithm in order to jointly minimize the average error across multiple tasks and demonstrate how this can be used to greatly speed up k-fold cross-validation. Lastly, we propose an adaptation of a recently developed acquisition function, en- tropy search, to the cost-sensitive, multi-task setting. We demonstrate the utility of this new acquisition function by leveraging a small dataset to explore hyper- parameter settings for a large dataset. Our algorithm dynamically chooses which dataset to query in order to yield the most information per unit cost.},
author = {Swersky, Kevin and Snoek, Jasper and Adams, Ryan P.},
booktitle = {Advances in Neural Information Processing Systems 26},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Swersky, Snoek, Adams - 2013 - Multi-Task Bayesian Optimization.pdf:pdf},
pages = {2004--2012},
title = {{Multi-Task Bayesian Optimization}},
url = {http://papers.nips.cc/paper/5086-multi-task-bayesian-optimization.pdf},
year = {2013}
}
@article{Ataman2007,
author = {Ataman, Kaan},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ataman - 2007 - Learning to rank by maximizing the AUC with linear programming for problems with binary output.pdf:pdf},
title = {{Learning to rank by maximizing the AUC with linear programming for problems with binary output}},
year = {2007}
}
@article{Lam2008,
author = {Lam, Patrick},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Lam - 2008 - MCMC Methods Gibbs Sampling and the Metropolis-Hastings Algorithm.pdf:pdf},
title = {{MCMC Methods : Gibbs Sampling and the Metropolis-Hastings Algorithm}},
year = {2008}
}
@article{Wolpert1997,
abstract = {A framework is developed to explore the connection between
effective optimization algorithms and the problems they are solving. A
number of \“no free lunch\” (NFL) theorems are presented which
establish that for any algorithm, any elevated performance over one
class of problems is offset by performance over another class. These
theorems result in a geometric interpretation of what it means for an
algorithm to be well suited to an optimization problem. Applications of
the NFL theorems to information-theoretic aspects of optimization and
benchmark measures of performance are also presented. Other issues
addressed include time-varying optimization problems and a priori
\“head-to-head\” minimax distinctions between optimization
algorithms, distinctions that result despite the NFL theorems' enforcing
of a type of uniformity over all algorithms},
author = {Wolpert, David H. and Macready, William G.},
doi = {10.1109/4235.585893},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wolpert, Macready - 1997 - No free lunch theorems for optimization.pdf:pdf},
isbn = {1089778X},
issn = {1089778X},
journal = {IEEE Transactions on Evolutionary Computation},
keywords = {Evolutionary algorithms,Information theory,Optimization},
number = {1},
pages = {67--82},
pmid = {97063701362},
title = {{No free lunch theorems for optimization}},
volume = {1},
year = {1997}
}
@article{Liaw2002,
abstract = {Recently there has been a lot of interest in “ensemble learning” — methods that generate many classifiers and aggregate their results. Two well-known methods are boosting (see, e.g., Shapire et al., 1998) and bagging Breiman (1996) of classification trees. In boosting, successive trees give extra weight to points incorrectly predicted by earlier predictors. In the end, a weighted vote is taken for prediction. In bagging, successive trees do not depend on earlier trees — each is independently constructed using a bootstrap sample of the data set. In the end, a simple majority vote is taken for prediction.},
archivePrefix = {arXiv},
arxivId = {1609-3631},
author = {Liaw, Andy and Wiener, Matthew},
doi = {10.1177/154405910408300516},
eprint = {1609-3631},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Liaw, Wiener - 2002 - Classification and Regression by randomForest.pdf:pdf},
isbn = {1609-3631},
issn = {16093631},
journal = {R news},
number = {December},
pages = {18--22},
pmid = {21196786},
title = {{Classification and Regression by randomForest}},
volume = {2},
year = {2002}
}
@article{Hand1997,
abstract = {Credit scoring is the term used to describe formal statistical methods used for classifying applicants for credit into 'good' and 'bad' risk classes. Such methods have become in- creasingly important with the dramatic growth in consumer credit in recent years. A wide range of statistical methods has been applied, though the literature available to the public is limited for reasons of commercial confidentiality. Particular problems arising in the credit scoring context are examined and the statistical methods which have been applied are reviewed.},
author = {Hand, David J. and Henley, W. E.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hand, Henley - 1997 - Statistical Classification Methods in Consumer Credit Scoring a Review.pdf:pdf},
journal = {Journal of the Royal Statistical Society},
keywords = {classification,consumer loans,credit control,credit scoring,discriminant analysis,finance,reject inference,risk assessment},
number = {3},
pages = {523--541},
title = {{Statistical Classification Methods in Consumer Credit Scoring: a Review}},
volume = {160},
year = {1997}
}
@article{Xu2014,
author = {Xu, Wei and Chen, Xi and Coleman, Thomas F},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Xu, Chen, Coleman - 2014 - The Efficient Application of Automatic Differentiation for Computing Gradients in Financial Applications ∗.pdf:pdf},
pages = {1--24},
title = {{The Efficient Application of Automatic Differentiation for Computing Gradients in Financial Applications ∗}},
year = {2014}
}
@article{Park2008,
abstract = {The Lasso estimate for linear regression parameters can be interpreted as a Bayesian posterior mode estimate when the regression parameters have independent Laplace (i.e., double-exponential) priors. Gibbs sampling from this posterior is possible using an expanded hierarchy with conjugate normal priors for the regression parameters and independent exponential priors on their variances. A connection with the inverse-Gaussian distribution provides tractable full conditional distributions. The Bayesian Lasso provides interval estimates (Bayesian credible intervals) that can guide variable selection. Moreover, the structure of the hierarchical model provides both Bayesian and likelihood methods for selecting the Lasso parameter. Slight modifications lead to Bayesian versions of other Lasso-related estimation methods, including bridge regression and a robust variant.},
author = {Park, Trevor and Casella, George},
doi = {10.1198/016214508000000337},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Park, Casella - 2008 - The Bayesian Lasso.pdf:pdf},
isbn = {0162-1459},
issn = {0162-1459},
journal = {Journal of the American Statistical Association},
keywords = {empirical bayes,gibbs sampler,hierarchical model,inverse gaussian,linear regression,penalized regression,scale},
number = {482},
pages = {681--686},
pmid = {21156729},
title = {{The Bayesian Lasso}},
volume = {103},
year = {2008}
}
@article{Brefeld2005,
abstract = {The area under the ROC curve (AUC) is a natural performance measure when the goal is to find a discriminative decision function. We present a rigorous derivation of an AUC maximizing Support Vector Machine; its optimization criterion is composed of a convex bound on the AUC and a margin term. The number of constraints in the optimization problem grows quadratically in the number of examples. We discuss an approximation for large data sets that clusters the constraints. Our experiments show that the AUC maximizing Support Vector Machine does in fact lead to higher AUC values.},
author = {Brefeld, Ulf and Scheffer, Tobias},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Brefeld, Scheffer - 2005 - AUC maximizing support vector learning.pdf:pdf},
journal = {\ldots workshop on ROC Analysis in Machine Learning},
number = {1},
title = {{AUC maximizing support vector learning}},
url = {http://scholar.google.com/scholar?hl=en\&btnG=Search\&q=intitle:AUC+Maximizing+Support+Vector+Learning\#0},
year = {2005}
}
@article{Shotton2013a,
abstract = {Randomized decision trees and forests have a rich history in machine learning and have seen considerable success in application, perhaps particularly so for com- puter vision. However, they face a fundamental limitation: given enough data, the number of nodes in decision trees will grow exponentially with depth. For certain applications, for example on mobile or embedded processors, memory is a limited resource, and so the exponential growth of trees limits their depth, and thus their potential accuracy. This paper proposes decision jungles, revisiting the idea of ensembles of rooted decision directed acyclic graphs (DAGs), and shows these to be compact and powerful discriminative models for classification. Unlike conventional decision trees that only allow one path to every node, a DAG in a decision jungle allows multiple paths from the root to each leaf. We present and compare two new node merging algorithms that jointly optimize both the features and the structure of the DAGs efficiently. During training, node splitting and node merging are driven by the minimization of exactly the same objective function, here the weighted sum of entropies at the leaves. Results on varied datasets show that, compared to decision forests and several other baselines, decision jungles require dramatically less memory while considerably improving generalization. 1},
author = {Shotton, Jamie and Nowozin, Sebastian and Sharp, Toby and Winn, John and Kohli, Pushmeet and Criminisi, Antonio},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Shotton et al. - 2013 - Decision Jungles Compact and Rich Models for Classification.pdf:pdf},
journal = {Advances in Neural \ldots},
pages = {1--9},
title = {{Decision Jungles: Compact and Rich Models for Classification}},
url = {http://papers.nips.cc/paper/5199-decision-jungles-compact-and-rich-models-for-classification},
year = {2013}
}
@article{Smith2002,
abstract = {This tutorial is designed to give the reader an understanding of Principal Components Analysis (PCA). PCA is a useful statistical technique that has found application in fields such as face recognition and image compression, and is a common technique for finding patterns in data of high dimension. Before getting to a description of PCA, this tutorial first introduces mathematical concepts that will be used in PCA. It covers standard deviation, covariance, eigenvec- tors and eigenvalues. This background knowledge is meant to make the PCA section very straightforward, but can be skipped if the concepts are already familiar. There are examples all the way through this tutorial that are meant to illustrate the concepts being discussed. If further information is required, the mathematics textbook Elementary Linear Algebra 5e by Howard Anton, Publisher JohnWiley \& Sons Inc, ISBN 0-471-85223-6 is a good source of information regarding the mathematical back- ground. 1},
author = {Smith, Lindsay I},
doi = {10.1080/03610928808829796},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Smith - 2002 - A tutorial on Principal Components Analysis Introduction.pdf:pdf},
isbn = {0471852236},
issn = {03610926},
journal = {Statistics},
pages = {52},
pmid = {16765218},
title = {{A tutorial on Principal Components Analysis Introduction}},
url = {http://www.mendeley.com/research/computational-genome-analysis-an-introduction-statistics-for-biology-and-health/},
volume = {51},
year = {2002}
}
@techreport{Goncalves2015,
author = {Gon\c{c}alves, Paulo},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Gon\c{c}alves - 2015 - Echantillonneurs de Monte Carlo Application \`{a} l’Estimation Statistique Bay\'{e}sienne.pdf:pdf},
title = {{Echantillonneurs de Monte Carlo Application \`{a} l’Estimation Statistique Bay\'{e}sienne}},
year = {2015}
}
@article{Nowozin2012,
abstract = {Ensembles of classification and regression trees remain popular machine learning methods because they define flexible non- parametric models that predict well and are computationally efficient both during train- ing and testing. During induction of deci- sion trees one aims to find predicates that are maximally informative about the pre- diction target. To select good predicates most approaches estimate an information- theoretic scoring function, the information gain, both for classification and regression problems. We point out that the common es- timation procedures are biased and show that by replacing them with improved estimators of the discrete and the differential entropy we can obtain better decision trees. In effect our modifications yield improved predictive per- formance and are simple to implement in any decision tree code.},
archivePrefix = {arXiv},
arxivId = {1206.4620},
author = {Nowozin, Sebastian},
eprint = {1206.4620},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Nowozin - 2012 - Improved Information Gain Estimates for Decision Tree Induction.pdf:pdf},
isbn = {978-1-4503-1285-1},
journal = {Proceedings of the 29th International Conference on Machine Learning (ICML-12)},
pages = {297--304},
title = {{Improved Information Gain Estimates for Decision Tree Induction}},
year = {2012}
}
@article{Hinton,
author = {Hinton, Geoffrey and Vinyals, Oriol and Dean, Jeffrey},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hinton, Vinyals, Dean - Unknown - Dark knowledge.pdf:pdf},
title = {{Dark knowledge}}
}
@techreport{Jahrer2009a,
author = {T\"{o}scher, Andreas and Jahrer, Michael and Bell, Robert M.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/T\"{o}scher, Jahrer, Bell - 2009 - The BigChaos Solution to the Netflix Grand Prize.pdf:pdf},
pages = {1--52},
title = {{The BigChaos Solution to the Netflix Grand Prize}},
year = {2009}
}
@article{Stefanowski,
author = {Stefanowski, Jerzy},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Stefanowski - Unknown - Random forest.pdf:pdf},
title = {{Random forest}}
}
@article{Wilson1997,
abstract = {Instance-based learning techniques typically handle continuous and linear input values well, but often do not handle nominal input attributes appropriately. The Value Difference Metric (VDM) was designed to find reasonable distance values between nominal attribute values, but it largely ignores continuous attributes, requiring discretization to map continuous values into nominal values. This paper proposes three new heterogeneous distance functions, called the Heterogeneous Value Difference Metric (HVDM), the Interpolated Value Difference Metric (IVDM), and the Windowed Value Difference Metric (WVDM). These new distance functions are designed to handle applications with nominal attributes, continuous attributes, or both. In experiments on 48 applications the new distance metrics achieve higher classification accuracy on average than three previous distance functions on those datasets that have both nominal and continuous attributes.},
archivePrefix = {arXiv},
arxivId = {cs/9701101},
author = {Wilson, D. Randall and Martinez, Tony R.},
doi = {10.1613/jair.346},
eprint = {9701101},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wilson, Martinez - 1997 - Improved heterogeneous distance functions.pdf:pdf},
issn = {10769757},
journal = {Journal of Artificial Intelligence Research},
pages = {1--34},
primaryClass = {cs},
title = {{Improved heterogeneous distance functions}},
volume = {6},
year = {1997}
}
@article{Arlot2014,
abstract = {Random forests are a very effective and commonly used statistical method, but their full theoretical analysis is still an open problem. As a first step, simplified models such as purely random forests have been introduced, in order to shed light on the good performance of random forests. In this paper, we study the approximation error (the bias) of some purely random forest models in a regression framework, focusing in particular on the influence of the number of trees in the forest. Under some regularity assumptions on the regression function, we show that the bias of an infinite forest decreases at a faster rate (with respect to the size of each tree) than a single tree. As a consequence, infinite forests attain a strictly better risk rate (with respect to the sample size) than single trees. Furthermore, our results allow to derive a minimum number of trees sufficient to reach the same rate as an infinite forest. As a by-product of our analysis, we also show a link between the bias of purely random forests and the bias of some kernel estimators. 1},
archivePrefix = {arXiv},
arxivId = {arXiv:1407.3939v1},
author = {Arlot, Sylvain and Genuer, Robin},
eprint = {arXiv:1407.3939v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Arlot, Genuer - 2014 - Analysis of purely random forests bias.pdf:pdf},
number = {2008},
title = {{Analysis of purely random forests bias}},
year = {2014}
}
@article{Bengio2013,
abstract = {The success of machine learning algorithms generally depends on data representation, and we hypothesize that this is because different representations can entangle and hide more or less the different explanatory factors of variation behind the data. Although specific domain knowledge can be used to help design representations, learning with generic priors can also be used, and the quest for AI is motivating the design of more powerful representation-learning algorithms implementing such priors. This paper reviews recent work in the area of unsupervised feature learning and deep learning, covering advances in probabilistic models, auto-encoders, manifold learning, and deep networks. This motivates longer-term unanswered questions about the appropriate objectives for learning good representations, for computing representations (i.e., inference), and the geometrical connections between representation learning, density estimation and manifold learning.},
archivePrefix = {arXiv},
arxivId = {1206.5538},
author = {Bengio, Yoshua and Courville, Aaron and Vincent, Pascal},
doi = {10.1109/TPAMI.2013.50},
eprint = {1206.5538},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bengio, Courville, Vincent - 2013 - Representation learning A review and new perspectives.pdf:pdf},
isbn = {0162-8828 VO - 35},
issn = {01628828},
journal = {Pattern Analysis and \ldots},
number = {1993},
pages = {1--30},
pmid = {23459267},
title = {{Representation learning: A review and new perspectives}},
url = {http://arxiv.org/abs/1206.5538$\backslash$nhttp://ieeexplore.ieee.org/xpls/abs\_all.jsp?arnumber=6472238},
year = {2013}
}
@book{Vapnik2000,
abstract = {The aim of this book is to discuss the fundamental ideas which lie behind the statistical theory of learning and generalization. It considers learning as a general problem of function estimation based on empirical data. Omitting proofs and technical details, the author concentrates on discussing the main results of learning theory and their connections to fundamental problems in statistics. These include: * the setting of learning problems based on the model of minimizing the risk functional from empirical data * a comprehensive analysis of the empirical risk minimization principle including necessary and sufficient conditions for its consistency * non-asymptotic bounds for the risk achieved using the empirical risk minimization principle * principles for controlling the generalization ability of learning machines using small sample sizes based on these bounds * the Support Vector methods that control the generalization ability when estimating function using small sample size. The second edition of the book contains three new chapters devoted to further development of the learning theory and SVM techniques. These include: * the theory of direct method of learning based on solving multidimensional integral equations for density, conditional probability, and conditional density estimation * a new inductive principle of learning. Written in a readable and concise style, the book is intended for statisticians, mathematicians, physicists, and computer scientists. Vladimir N. Vapnik is Technology Leader AT\&T Labs-Research and Professor of London University. He is one of the founders of statistical learning theory, and the author of seven books published in English, Russian, German, and Chinese.},
author = {Vapnik, Vladimir},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Vapnik - 2000 - The Nature of Statistical Learning Theory.pdf:pdf},
isbn = {0387987800},
pages = {314},
title = {{The Nature of Statistical Learning Theory}},
url = {http://books.google.com/books?hl=es\&id=sna9BaxVbj8C\&pgis=1$\backslash$nhttp://infoscience.epfl.ch/record/82790/files/com02-04.pdf},
year = {2000}
}
@article{Aronszajn1950,
author = {Aronszajn, N.},
doi = {10.1090/S0002-9947-1950-0051437-7},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Aronszajn - 1950 - Theory of reproducing kernels.pdf:pdf},
issn = {0002-9947},
journal = {Transactions of the American Mathematical Society},
keywords = {RKHS},
mendeley-tags = {RKHS},
month = mar,
number = {3},
pages = {337--337},
title = {{Theory of reproducing kernels}},
url = {http://www.ams.org/tran/1950-68-03/S0002-9947-1950-0051437-7/},
volume = {68},
year = {1950}
}
@techreport{Gonzalez,
author = {Gonzalez, Pierre-louis},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Gonzalez - Unknown - Segmentation.pdf:pdf},
title = {{Segmentation}}
}
@article{Barros2014,
abstract = {Decision-tree induction algorithms are widely used in knowledge discovery and data mining, specially in scenarios where model comprehensibility is desired. A variation of the traditional univariate approach is the so-called oblique decision tree, which allows multivariate tests in its non-terminal nodes. Oblique decision trees can model decision boundaries that are oblique to the attribute axes, whereas univariate trees can only perform axis-parallel splits. The vast majority of the oblique and univariate decision-tree induction algorithms employ a top-down strategy for growing the tree, relying on an impurity-based measure for splitting nodes. In this paper, we propose BUTIF-a novel Bottom- Up Oblique Decision- Tree Induction Framework. BUTIF does not rely on an impurity-measure for dividing nodes, since the data resulting from each split is known a priori. For generating the initial leaves of the tree and the splitting hyperplanes in its internal nodes, BUTIF allows the adoption of distinct clustering algorithms and binary classifiers, respectively. It is also capable of performing embedded feature selection, which may reduce the number of features in each hyperplane, thus improving model comprehension. Different from virtually every top-down decision-tree induction algorithm, BUTIF does not require the further execution of a pruning procedure in order to avoid overfitting, due to its bottom-up nature that does not overgrow the tree. We compare distinct instances of BUTIF to traditional univariate and oblique decision-tree induction algorithms. Empirical results show the effectiveness of the proposed framework. © 2014 Elsevier B.V.},
author = {Barros, Rodrigo C. and Jaskowiak, Pablo a. and Cerri, Ricardo and {De Carvalho}, Andre C P L F},
doi = {10.1016/j.neucom.2013.01.067},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Barros et al. - 2014 - A framework for bottom-up induction of oblique decision trees.pdf:pdf},
issn = {18728286},
journal = {Neurocomputing},
keywords = {Bottom-up induction,Clustering,Oblique decision trees},
number = {August 2015},
pages = {3--12},
title = {{A framework for bottom-up induction of oblique decision trees}},
volume = {135},
year = {2014}
}
@article{Ram2011,
abstract = {In this paper we develop density estimation trees (DETs), the natural analog of classification trees and regression trees, for the task of density estimation. We consider the estimation of a joint probability density function of a d-dimensional random vector X and define a piecewise constant estimator structured as a decision tree. The integrated squared error is minimized to learn the tree. We show that the method is nonparametric: under standard conditions of nonparametric density estimation, DETs are shown to be asymptotically consistent. In addition, being decision trees, DETs perform automatic feature selection. They empirically exhibit the interpretability, adaptability and feature selection properties of supervised decision trees while incurring slight loss in accuracy over other nonparametric density estimators. Hence they might be able to avoid the curse of dimensionality if the true density is sparse in dimensions. We believe that density estimation trees provide a new tool for exploratory data analysis with unique capabilities.},
author = {Ram, Parikshit and Ga, Atlanta and Gray, Alexander G and Models, I Pattern Recognition and Probability, G},
doi = {10.1145/2020408.2020507},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ram et al. - 2011 - Density Estimation Trees.pdf:pdf},
isbn = {9781450308137},
journal = {Methods},
keywords = {data analysis,decision trees,density estimation},
pages = {627--635},
title = {{Density Estimation Trees}},
url = {http://users.cis.fiu.edu/~lzhen001/activities/KDD2011Program/docs/p627.pdf},
year = {2011}
}
@article{Ogutu2011,
abstract = {Genomic selection (GS) involves estimating breeding values using molecular markers spanning the entire genome. Accurate prediction of genomic breeding values (GEBVs) presents a central challenge to contemporary plant and animal breeders. The existence of a wide array of marker-based approaches for predicting breeding values makes it essential to evaluate and compare their relative predictive performances to identify approaches able to accurately predict breeding values. We evaluated the predictive accuracy of random forests (RF), stochastic gradient boosting (boosting) and support vector machines (SVMs) for predicting genomic breeding values using dense SNP markers and explored the utility of RF for ranking the predictive importance of markers for pre-screening markers or discovering chromosomal locations of QTLs.},
author = {Ogutu, Joseph O and Piepho, Hans-Peter and Schulz-Streeck, Torben},
doi = {10.1186/1753-6561-5-S3-S11},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ogutu, Piepho, Schulz-Streeck - 2011 - A comparison of random forests, boosting and support vector machines for genomic selection.pdf:pdf},
isbn = {1753-6561 (Electronic)$\backslash$r1753-6561 (Linking)},
issn = {1753-6561},
journal = {BMC proceedings},
number = {Suppl 3},
pages = {S11},
pmid = {21624167},
publisher = {BioMed Central Ltd},
title = {{A comparison of random forests, boosting and support vector machines for genomic selection.}},
url = {http://www.biomedcentral.com/1753-6561/5/S3/S11},
volume = {5 Suppl 3},
year = {2011}
}
@inproceedings{Menze2011,
abstract = {In his original paper on random forests, Breiman proposed two different decision tree ensembles: one generated from “orthogonal” trees with thresholds on individual features in every split, and one from “oblique” trees separating the feature space by randomly oriented hy- perplanes. In spite of a rising interest in the random forest framework, however, ensembles built from orthogonal trees (RF) have gained most, if not all, attention so far. In the present work we propose to employ “oblique” random forests (oRF) built from multivariate trees which explicitly learn optimal split directions at internal nodes using linear discriminative models, rather than using random coefficients as the original oRF. This oRF outper- forms RF, as well as other classifiers, on nearly all data sets but those with discrete factorial features. Learned node models perform distinc- tively better than random splits. An oRF feature importance score shows to be preferable over standard RF feature importance scores such as Gini or permutation importance. The topology of the oRF decision space ap- pears to be smoother and better adapted to the data, resulting in im- proved generalization performance. Overall, the oRF propose here may be preferred over standard RF on most learning tasks involving numeri- cal and spectral data.},
author = {Menze, Bjoern H. and Kelm, B. Michael and Splitthoff, Daniel N. and Koethe, Ullrich and Hamprecht, Fred A.},
booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
doi = {10.1007/978-3-642-23783-6\_29},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Menze et al. - 2011 - On oblique random forests.pdf:pdf},
isbn = {9783642237829},
issn = {03029743},
number = {PART 2},
pages = {453--469},
title = {{On oblique random forests}},
volume = {6912 LNAI},
year = {2011}
}
@article{Lin2013,
abstract = {A class-imbalanced classifier is a decision rule to predict the class membership of new samples from an available data set where the class sizes differ considerably. When the class sizes are very different, most standard classification algorithms may favor the larger (majority) class resulting in poor accuracy in the minority class prediction. A class-imbalanced classifier typically modifies a standard classifier by a correction strategy or by incorporating a new strategy in the training phase to account for differential class sizes. This article reviews and evaluates some most important methods for class prediction of high-dimensional imbalanced data. The evaluation addresses the fundamental issues of the class-imbalanced classification problem: imbalance ratio, small disjuncts and overlap complexity, lack of data and feature selection. Four class-imbalanced classifiers are considered. The four classifiers include three standard classification algorithms each coupled with an ensemble correction strategy and one support vector machines (SVM)-based correction classifier. The three algorithms are (i) diagonal linear discriminant analysis (DLDA), (ii) random forests (RFs) and (ii) SVMs. The SVM-based correction classifier is SVM threshold adjustment (SVM-THR). A Monte-Carlo simulation and five genomic data sets were used to illustrate the analysis and address the issues. The SVM-ensemble classifier appears to perform the best when the class imbalance is not too severe. The SVM-THR performs well if the imbalance is severe and predictors are highly correlated. The DLDA with a feature selection can perform well without using the ensemble correction.},
author = {Lin, Wei Jiun and Chen, James J.},
doi = {10.1093/bib/bbs006},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Lin, Chen - 2013 - Class-imbalanced classifiers for high-dimensional data.pdf:pdf},
isbn = {1477-4054},
issn = {14675463},
journal = {Briefings in Bioinformatics},
keywords = {Class-imbalanced prediction,Feature selection,Lack of data,Performance metrics,Threshold adjustment,Under-sampling ensemble},
number = {1},
pages = {13--26},
pmid = {22408190},
title = {{Class-imbalanced classifiers for high-dimensional data}},
volume = {14},
year = {2013}
}
@article{Furry,
abstract = {We introduce canonical correlation forests (CCFs), a new decision tree ensem- ble method for classification. Individual canonical correlation trees are binary decision trees with hyperplane splits based on canonical correlation components. Unlike axis-aligned alternatives, the decision surfaces of CCFs are not restricted to the coordinate system of the input features and therefore more naturally rep- resents data with correlation between the features. Additionally we introduce a novel alternative to bagging, the projection bootstrap, which maintains use of the full dataset in selecting split points. CCFs do not require parameter tuning and our experiments show that they significantly out-perform axis-aligned random forests and other state-of-the-art tree ensemble methods. 1},
archivePrefix = {arXiv},
arxivId = {1507.05444},
author = {Rainforth, Tom and Wood, Frank},
eprint = {1507.05444},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Rainforth, Wood - Unknown - Canonical Correlation Forests.pdf:pdf},
pages = {1--13},
title = {{Canonical Correlation Forests}}
}
@article{Chipman2008,
abstract = {We develop a Bayesian "sum-of-trees" model where each tree is constrained by a regularization prior to be a weak learner, and fitting and inference are accomplished via an iterative Bayesian backfitting MCMC algorithm that generates samples from a posterior. Effectively, BART is a nonparametric Bayesian regression approach which uses dimensionally adaptive random basis elements. Motivated by ensemble methods in general, and boosting algorithms in particular, BART is defined by a statistical model: a prior and a likelihood. This approach enables full posterior inference including point and interval estimates of the unknown regression function as well as the marginal effects of potential predictors. By keeping track of predictor inclusion frequencies, BART can also be used for model-free variable selection. BART's many features are illustrated with a bake-off against competing methods on 42 different data sets, with a simulation experiment and on a drug discovery classification problem.},
archivePrefix = {arXiv},
arxivId = {0806.3286},
author = {Chipman, Hugh a. and George, Edward I. and McCulloch, Robert E.},
doi = {10.1214/09-AOAS285},
eprint = {0806.3286},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Chipman, George, McCulloch - 2008 - BART Bayesian additive regression trees.pdf:pdf},
isbn = {1932-6157},
issn = {19326157},
journal = {Annals of Applied Statistics},
keywords = {Bayesian backfitting,Boosting,CART,Classification,Ensemble,MCMC,Nonparametric regression,Probit model,Random basis,Regularizatio,Sum-of-trees model,Variable selection,Weak learner},
number = {1},
pages = {266--298},
title = {{BART: Bayesian additive regression trees}},
volume = {6},
year = {2008}
}
@article{Capriotti2010,
abstract = {We show how Algorithmic Differentiation can be used to implement efficiently the Pathwise Deriva- tive method for the calculation of option sensitivities with Monte Carlo. The main practical diffi- culty of the Pathwise Derivative method is that it requires the differentiation of the payout func- tion. For the type of structured options for which Monte Carlo simulations are usually employed, these derivatives are typically cumbersome to calculate analytically, and too time consuming to evaluate with standard finite-differences approaches. In this paper we address this problem and show how Algorithmic Differentiation can be employed to calculate very efficiently and with ma- chine precision accuracy these derivatives. We illustrate the basic workings of this computational technique by means of simple examples, and we demonstrate with several numerical tests how the Pathwise Derivative method combined with Algorithmic Differentiation – especially in the adjoint mode – can provide speed-ups of several orders of magnitude with respect to standard methods},
author = {Capriotti, Luca},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Capriotti - 2010 - Fast Greeks by Algorithmic Differentiation.pdf:pdf},
journal = {SSRN},
keywords = {algorithmic differentiation,derivatives pricing,monte carlo simulations},
number = {3},
pages = {1--15},
title = {{Fast Greeks by Algorithmic Differentiation}},
url = {http://papers.ssrn.com/sol3/papers.cfm?abstract\_id=1619626$\backslash$nhttp://www.luca-capriotti.net/pdfs/Finance/GD11LucaCapriotti.pdf},
volume = {14},
year = {2010}
}
@techreport{Orhan2014,
author = {Orhan, Emin},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Orhan - 2014 - Cover’s Function Counting Theorem ( 1965 ).pdf:pdf},
number = {1965},
pages = {1--2},
title = {{Cover’s Function Counting Theorem ( 1965 )}},
year = {2014}
}
@misc{Buuren2015,
author = {Buuren, Stef Van},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Buuren - 2015 - Package ‘ mice ’.pdf:pdf},
title = {{Package ‘ mice ’}},
year = {2015}
}
@article{Wolpert1992,
abstract = {: This paper introduces stacked generalization, a scheme for minimizing the generalization error rate of one or more generalizers. Stacked generalization works by deducing the biases of the generalizer(s) with respect to a provided learning set. This deduction proceeds by generalizing in a second space whose inputs are (for example) the guesses of the original generalizers when taught with part of the learning set and trying to guess the rest of it, and whose output is (for example) the correct guess. When used with multiple generalizers, stacked generalization can be seen as a more sophisticated version of cross-validation, exploiting a strategy more sophisticated than cross-validation 's crude winner-takes-all for combining the individual generalizers. When used with a single generalizer, stacked generalization is a scheme for estimating (and then correcting for) the error of a generalizer which has been trained on a particular learning set and then asked a particular question. After...},
author = {Wolpert, David H.},
doi = {10.1016/S0893-6080(05)80023-1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wolpert - 1992 - Stacked generalization.pdf:pdf},
isbn = {0893-6080},
issn = {08936080},
journal = {Neural Networks},
number = {2},
pages = {241--259},
pmid = {17947137},
title = {{Stacked generalization}},
volume = {5},
year = {1992}
}
@article{Thakur2015,
abstract = {In this paper, we propose AutoCompete, a highly automated machine learning framework for tackling machine learning competitions. This framework has been learned by us, validated and improved over a period of more than two years by participating in online machine learning competitions. It aims at minimizing human interference required to build a first useful predictive model and to assess the practical difficulty of a given machine learning challenge. The proposed system helps in identifying data types, choosing a machine learn- ing model, tuning hyper-parameters, avoiding over-fitting and optimization for a provided evaluation metric. We also observe that the proposed system produces better (or comparable) results with less runtime as compared to other approaches.},
archivePrefix = {arXiv},
arxivId = {1507.02188},
author = {Thakur, Abhishek and Krohn-Grimberghe, Artus},
eprint = {1507.02188},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Thakur, Krohn-Grimberghe - 2015 - AutoCompete A Framework for Machine Learning Competition.pdf:pdf},
keywords = {auto-machine learning,predictive modelling},
title = {{AutoCompete: A Framework for Machine Learning Competition}},
url = {http://arxiv.org/abs/1507.02188},
year = {2015}
}
@article{Gneiting2007,
abstract = {Scoring rules assess the quality of probabilistic forecasts, by assigning a numerical score based on the predictive distribution and on the event or value that materializes. A scoring rule is proper if the forecaster maximizes the expected score for an observation drawn from the distribution F if he or she issues the probabilistic forecast F, rather than G?=F. It is strictly proper if the maximum is unique. In prediction problems, proper scoring rules encourage the forecaster tomake careful assessments and to be honest. In estimation problems, strictly proper scoring rules provide attractive loss and utility functions that can be tailored to the problem at hand. This article reviews and develops the theory of proper scoring rules on general probability spaces, and proposes and discusses examples thereof. Proper scoring rules derive from convex functions and relate to information measures, entropy functions, and Bregman divergences. In the case of categorical variables, we prove a rigorous version of the Savage representation. Examples of scoring rules for probabilistic forecasts in the form of predictive densities include the logarithmic, spherical, pseudospherical, and quadratic scores. The continuous ranked probability score applies to probabilistic forecasts that take the form of predictive cumulative distribution functions. It generalizes the absolute error and forms a special case of a new and very general type of score, the energy score. Like many other scoring rules, the energy score admits a kernel representation in terms of negative definite functions, with links to inequalities of Hoeffding type, in both univariate and multivariate settings. Proper scoring rules for quantile and interval forecasts are also discussed.We relate proper scoring rules to Bayes factors and to cross-validation, and propose a novel form of cross-validation known as random-fold cross-validation. A case study on probabilistic weather forecasts in the North American Pacific Northwest illustrates the importance of propriety. We note optimum score approaches to point and quantile estimation, and propose the intuitively appealing interval score as a utility function in interval estimation that addresses width as well as coverage.},
author = {Gneiting, Tilmann and Raftery, Adrian E},
doi = {10.1198/016214506000001437},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Gneiting, Raftery - 2007 - Strictly Proper Scoring Rules, Prediction, and Estimation.pdf:pdf},
isbn = {0162-1459},
issn = {0162-1459},
journal = {Journal of the American Statistical Association},
number = {477},
pages = {359--378},
title = {{Strictly Proper Scoring Rules, Prediction, and Estimation}},
volume = {102},
year = {2007}
}
@article{Nan2015,
abstract = {We seek decision rules for prediction-time cost reduction, where complete data is available for training, but during prediction-time, each feature can only be acquired for an additional cost. We propose a novel random forest algorithm to min- imize prediction error for a user-specified aver- age feature acquisition budget. While random forests yield strong generalization performance, they do not explicitly account for feature costs and furthermore require low correlation among trees, which amplifies costs. Our random for- est grows trees with low acquisition cost and high strength based on greedy minimax cost- weighted-impurity splits. Theoretically, we es- tablish near-optimal acquisition cost guarantees for our algorithm. Empirically, on a number of benchmark datasets we demonstrate compet- itive accuracy-cost curves against state-of-the-art prediction-time},
author = {Nan, Feng and Wang, Joseph and Saligrama, Venkatesh},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Nan, Wang, Saligrama - 2015 - Feature-Budgeted Random Forest.pdf:pdf},
title = {{Feature-Budgeted Random Forest}},
volume = {37},
year = {2015}
}
@article{Cortes2003,
abstract = {The area under an ROC curve (AUC) is a criterion used in many applications to measure the quality of a classification algorithm. However, the objective function optimized in most of these algorithms is the error rate and not the AUC value. We give a detailed statistical analysis of the relationship between the AUC and the error rate, including the first exact expression of the expected value and the variance of the AUC for a fixed error rate. Our results show that the average AUC is monotonically increasing as a function of the classification accuracy, but that the standard deviation for uneven distributions and higher error rates is noticeable. Thus, algorithms designed to minimize the error rate may not lead to the best possible AUC values. We show that, under certain conditions, the global function optimized by the RankBoost algorithm is exactly the AUC.We report the results of our experiments with RankBoost in several datasets demonstrating the benefits of an algorithmspecifically designed to globally optimize the AUC over other existing algorithms optimizing an approximation of the AUC or only locally optimizing the AUC. 1},
author = {Cortes, Corinna and Mohri, Mehryar},
doi = {10.1.1.9.3518},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cortes, Mohri - 2003 - AUC Optimization vs. Error Rate Minimization.pdf:pdf},
isbn = {0262201526},
issn = {10495258},
journal = {Advances in Neural Information Processing Systems},
pages = {313--320},
title = {{AUC Optimization vs. Error Rate Minimization.}},
url = {https://papers.nips.cc/paper/2518-auc-optimization-vs-error-rate-minimization.pdf},
year = {2003}
}
@article{Menon2009,
abstract = {Support Vector Machines (SVMs) are a very popular method for binary classification. Traditional training algorithms for SVMs, such as chunking and SMO},
author = {Menon, Aditya Krishna},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Menon - 2009 - Large-scale support vector machines algorithms and theory.pdf:pdf},
journal = {Research Exam, University of California, San Diego},
title = {{Large-scale support vector machines: algorithms and theory}},
url = {http://cseweb.ucsd.edu/~akmenon/ResearchExamTalk.pdf$\backslash$npapers2://publication/uuid/57F5C13E-738D-4EB1-9DA6-EC3D6E3B3761},
year = {2009}
}
@article{Snoek2012a,
abstract = {Machine learning algorithms frequently require careful tuning of model hyperparameters, regularization terms, and optimization parameters. Unfortunately, this tuning is often a "black art" that requires expert experience, unwritten rules of thumb, or sometimes brute-force search. Much more appealing is the idea of developing automatic approaches which can optimize the performance of a given learning algorithm to the task at hand. In this work, we consider the automatic tuning problem within the framework of Bayesian optimization, in which a learning algorithm's generalization performance is modeled as a sample from a Gaussian process (GP). The tractable posterior distribution induced by the GP leads to efficient use of the information gathered by previous experiments, enabling optimal choices about what parameters to try next. Here we show how the effects of the Gaussian process prior and the associated inference procedure can have a large impact on the success or failure of Bayesian optimization. We show that thoughtful choices can lead to results that exceed expert-level performance in tuning machine learning algorithms. We also describe new algorithms that take into account the variable cost (duration) of learning experiments and that can leverage the presence of multiple cores for parallel experimentation. We show that these proposed algorithms improve on previous automatic procedures and can reach or surpass human expert-level optimization on a diverse set of contemporary algorithms including latent Dirichlet allocation, structured SVMs and convolutional neural networks.},
archivePrefix = {arXiv},
arxivId = {1206.2944},
author = {Snoek, Jasper and Larochelle, Hugo and Adams, Ryan P. Rp},
eprint = {1206.2944},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Snoek, Larochelle, Adams - 2012 - Practical Bayesian Optimization of Machine Learning Algorithms.pdf:pdf},
isbn = {9781627480031},
journal = {arXiv preprint arXiv:1206.2944},
pages = {1--12},
title = {{Practical Bayesian Optimization of Machine Learning Algorithms}},
url = {http://arxiv.org/abs/1206.2944},
year = {2012}
}
@article{Taddy2015,
abstract = {We derive ensembles of decision trees through a nonparametric Bayesian model, allowing us to view random forests as samples from a posterior distribution. This insight provides large gains in interpretability, and motivates a class of Bayesian forest (BF) algorithms that yield small but reli- able performance gains. Based on the BF frame- work, we are able to show that high-level tree hi- erarchy is stable in large samples. This leads to an empirical Bayesian forest (EBF) algorithm for building approximate BFs on massive distributed datasets and we show that EBFs outperform sub- sampling based alternatives by a large margin. 1.},
archivePrefix = {arXiv},
arxivId = {arXiv:1502.02312v2},
author = {Taddy, Matt and Chen, Chun-sheng and Com, Chunschen Ebay and Wyle, Mitch},
eprint = {arXiv:1502.02312v2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Taddy et al. - 2015 - Bayesian and Empirical Bayesian Forests.pdf:pdf},
title = {{Bayesian and Empirical Bayesian Forests}},
volume = {37},
year = {2015}
}
@book{Bengio2015,
author = {Bengio, Yoshua and Goodfellow, Ian J. and Courville, Aaron},
publisher = {MIT Press},
title = {{Deep Learning}},
url = {http://www.iro.umontreal.ca/~bengioy/dlbook},
year = {2015}
}
@article{Bergstra2012,
abstract = {Grid search and manual search are the most widely used strategies for hyper-parameter optimiza- tion. This paper shows empirically and theoretically that randomly chosen trials are more efficient for hyper-parameter optimization than trials on a grid. Empirical evidence comes from a compar- ison with a large previous study that used grid search and manual search to configure neural net- works and deep belief networks. Compared with neural networks configured by a pure grid search, we find that random search over the same domain is able to find models that are as good or better within a small fraction of the computation time. Granting random search the same computational budget, random search finds better models by effectively searching a larger, less promising con- figuration space. Compared with deep belief networks configured by a thoughtful combination of manual search and grid search, purely random search over the same 32-dimensional configuration space found statistically equal performance on four of seven data sets, and superior performance on one of seven. A Gaussian process analysis of the function from hyper-parameters to validation set performance reveals that for most data sets only a few of the hyper-parameters really matter, but that different hyper-parameters are important on different data sets. This phenomenon makes grid search a poor choice for configuring algorithms for new data sets. Our analysis casts some light on why recent “High Throughput”methods achieve surprising success—they appear to search through a large number of hyper-parameters because most hyper-parameters do not matter much. We anticipate that growing interest in large hierarchical models will place an increasing burden on techniques for hyper-parameter optimization; this work shows that randomsearch is a natural base- line against which to judge progress in the development of adaptive (sequential) hyper-parameter optimization algorithms.},
author = {Bergstra, James and Bengio, Yoshua},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bergstra, Bengio - 2012 - Random Search for Hyper-Parameter Optimization.pdf:pdf},
isbn = {1532-4435},
issn = {1532-4435},
journal = {Journal ofMachine Learning Research},
keywords = {deep learning,global optimization,model selection,neural networks,response surface},
pages = {281--305},
title = {{Random Search for Hyper-Parameter Optimization}},
volume = {13},
year = {2012}
}
@article{Taylor2015,
author = {Taylor, Phillip and Griffiths, Nathan and Bhalerao, Abhir},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Taylor, Griffiths, Bhalerao - 2015 - Redundant Feature Selection using Permutation Methods.pdf:pdf},
journal = {ICML 2014 AutoML Workshop},
title = {{Redundant Feature Selection using Permutation Methods}},
year = {2015}
}
@article{Pittman2004,
abstract = {Classification tree models are flexible analysis tools which have the ability to evaluate interactions among predictors as well as generate predictions for responses of interest. We describe Bayesian analysis of a specific class of tree models in which binary response data arise from a retrospective case-control design. We are also particularly interested in problems with potentially very many candidate predictors. This scenario is common in studies concerning gene expression data, which is a key motivating example context. Innovations here include the introduction of tree models that explicitly address and incorporate the retrospective design, and the use of nonparametric Bayesian models involving Dirichlet process priors on the distributions of predictor variables. The model specification influences the generation of trees through Bayes' factor based tests of association that determine significant binary partitions of nodes during a process of forward generation of trees. We describe this constructive process and discuss questions of generating and combining multiple trees via Bayesian model averaging for prediction. Additional discussion of parameter selection and sensitivity is given in the context of an example which concerns prediction of breast tumour status utilizing high-dimensional gene expression data; the example demonstrates the exploratory/explanatory uses of such models as well as their primary utility in prediction. Shortcomings of the approach and comparison with alternative tree modelling algorithms are also discussed, as are issues of modelling and computational extensions.},
author = {Pittman, Jennifer and Huang, Erich and Nevins, Joseph and Wang, Quanli and West, Mike},
doi = {10.1093/biostatistics/kxh011},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Pittman et al. - 2004 - Bayesian analysis of binary prediction tree models for retrospectively sampled outcomes.pdf:pdf},
isbn = {1465-4644 (Print)$\backslash$n1465-4644 (Linking)},
issn = {14654644},
journal = {Biostatistics},
keywords = {Bayesian analysis,Binary classification tree,Bioinformatics,Case-control design,Metagenes,Molecular classification,Predictive classification,Retrospective sampling,Tree models},
number = {4},
pages = {587--601},
pmid = {15475421},
title = {{Bayesian analysis of binary prediction tree models for retrospectively sampled outcomes}},
volume = {5},
year = {2004}
}
@article{Rakotomamonjy2004,
abstract = {For many years now, there is a growing interest around ROC$\backslash$ncurve for characterizing machine learning performances. This$\backslash$nis particularly due to the fact that in real-world problems$\backslash$nmisclassification costs are not known and thus, ROC curve$\backslash$nand related metrics such as the Area Under ROC curve (AUC)$\backslash$ncan be a more meaningful performance measures. In this$\backslash$npaper, we propose a SVMs based algorithm for AUC$\backslash$nmaximization and show that under certain conditions this$\backslash$nalgorithm is related to 2-norm soft margin Support Vector$\backslash$nMachines. We present experiments that compare SVMs$\backslash$nperformances to those of other AUC maximization based$\backslash$nalgorithms and provide empirical analysis of SVMs behavior$\backslash$nwith regards to ROC- based metrics. Our main conclusion is$\backslash$nthat SVMs can maximize both AUC and accuracy.},
author = {Rakotomamonjy, Alain},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Rakotomamonjy - 2004 - Optimizing Area Under Roc Curve with SVMs.pdf:pdf},
journal = {Optimization},
title = {{Optimizing Area Under Roc Curve with SVMs}},
year = {2004}
}
@article{Bostr,
author = {Bostr, Henrik},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bostr - Unknown - Estimating Class Probabilities in Random Forests.pdf:pdf},
title = {{Estimating Class Probabilities in Random Forests}}
}
@article{Laurikkala2001,
abstract = {We studied three methods to improve identification of difficult small classes by balancing imbalanced class distribution with data reduction. The new method, neighborhood cleaning rule (NCL), outperformed simple random and one-sided selection methods in experiments with ten data sets. All reduction methods improved identification of small classes (20-30\%), but the differences were insignificant. However, significant differences in accuracies, true-positive rates and true-negative rates obtained with the 3-nearest neighbor method and C4.5 from the reduced data favored NCL. The results suggest that NCL is a useful method for improving the modeling of difficult small classes, and for building classifiers to identify these classes from the real-world data.},
author = {Laurikkala, Jorma},
doi = {10.1007/3-540-48229-6\_9},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Laurikkala - 2001 - Improving Identification of Difficult Small Classes by Balancing Class Distribution.pdf:pdf},
isbn = {0302-9743},
journal = {8th Conference on Artificial Intelligence in Medicine in Europe},
pages = {63--66},
title = {{Improving Identification of Difficult Small Classes by Balancing Class Distribution}},
year = {2001}
}
@article{Gubinelli2010,
author = {Massimiliano, Gubinelli},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Massimiliano - 2010 - III Th\'{e}or\`{e}mes limites pour les cha\^{\i}nes de Markov.pdf:pdf},
pages = {1--5},
title = {{III Th\'{e}or\`{e}mes limites pour les cha\^{\i}nes de Markov}},
year = {2010}
}
@article{Liu2009,
abstract = {Under-sampling is a class-imbalance learning method which uses only a subset of major class examples and thus is very efficient. The main deficiency is that many major class examples are ignored. We propose two algorithms to overcome the deficiency. EasyEnsemble samples several subsets from the major class, trains a learner using each of them, and combines the outputs of those learners. BalanceCascade is similar to EasyEnsemble except that it removes correctly classified major class examples of trained learners from further consideration. Experiments show that both of the proposed algorithms have better AUC scores than many existing class-imbalance learning methods. Moreover, they have approximately the same training time as that of under-sampling, which trains significantly faster than other methods.$\backslash$n$\backslash$nRecommended by Xin; Her experiment design is very good; Compare with my work; She seperates data sets into easy and hard tasks according to AUC value;},
author = {Liu, Xu-Ying and Wu, Jianxin and Zhou, Zhi-Hua},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Liu, Wu, Zhou - 2009 - Exploratory Undersampling for Class Imbalance Learning.pdf:pdf},
journal = {IEEE Transactions on Systems, Man and Cybernetics},
keywords = {Ensemble learning,sampling},
number = {2},
pages = {539--550},
title = {{Exploratory Undersampling for Class Imbalance Learning}},
volume = {39},
year = {2009}
}
@article{Blum,
abstract = {The empirical error on a test set, the hold-out esti- mate, often is a more reliable estimate of general- ization error than the observed error on the training set, the training estimate. K-fold cross validation is used in practice with the hope of being more ac- curate than the hold-out estimate without reducing the number of training examples. We argue that the k-fold estimate does in fact achieve this goal. Specifically, we showthat for any nontrivial learn- ing problem and learning algorithm that is insen- sitive to example ordering, the k-fold estimate is strictly more accurate than a single hold-out esti- mate on 1/k of the data, for ( is leave-one-out), based on its variance and all higher moments. Previous bounds were termed sanity- check because they compared the k-fold estimate to the training estimate and, further, restricted the ????????? ? ? VC dimension and required a notion of hypothesis stability [2]. In order to avoid these dependencies, we consider a k-fold hypothesis that is a random- ized combination or average of the individual hy- potheses. We introduce progressive validation as another pos- sible improvement on the hold-out estimate. This estimate of the generalization error is, in many ways, ? as good as that of a single hold-out, but it uses an average of half as many examples for testing. The procedure also involves a hold-out set, but after an example has been tested, it is added to the training set and the learning algorithm is rerun.},
author = {Blum, Avrim and Kalai, Adam and Langford, John},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Blum, Kalai, Langford - Unknown - Beating the Hold-Out † Bounds for K-Fold and Progressive Cross-Validation.pdf:pdf},
pages = {6--11},
title = {{Beating the Hold-Out : † Bounds for K-Fold and Progressive Cross-Validation}}
}
@article{Leistner,
abstract = {Random Forests (RFs) have become common- place in many computer vision applications. Their popularity is mainly driven by their high computa- tional efficiency during both training and evaluation while still being able to achieve state-of-the-art ac- curacy. This work extends the usage of Random Forests to Semi-Supervised Learning (SSL) problems. We show that traditional decision trees are optimizing multi- class margin maximizing loss functions. From this intuition, we develop a novel multi-class margin def- inition for the unlabeled data, and an iterative deter- ministic annealing-style training algorithm maximiz- ing both the multi-class margin of labeled and un- labeled samples. In particular, this allows us to use the predicted labels of the unlabeled data as addi- tional optimization variables. Furthermore, we pro- pose a control mechanism based on the out-of-bag error, which prevents the algorithm from degrada- tion if the unlabeled data is not useful for the task. Our experiments demonstrate state-of-the-art semi- supervised learning performance in typical machine learning problems and constant improvement using unlabeled data for the Caltech-101 object catego- rization task. 1.},
author = {Leistner, Christian and Saffari, Amir and Santner, Jakob and Bischof, Horst},
doi = {10.1109/ICCV.2009.5459198},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Leistner et al. - 2009 - Semi-Supervised Random Forests.pdf:pdf},
isbn = {9781424444205},
issn = {1550-5499},
journal = {Proceedings of the IEEE International Conference on Computer Vision},
pages = {506--513},
title = {{Semi-Supervised Random Forests}},
year = {2009}
}
@article{Rodriguez2006,
abstract = {We propose a method for generating classifier ensembles based on feature extraction. To create the training data for a base classifier, the feature set is randomly split into K subsets (K is a parameter of the algorithm) and Principal Component Analysis (PCA) is applied to each subset. All principal components are retained in order to preserve the variability information in the data. Thus, K axis rotations take place to form the new features for a base classifier. The idea of the rotation approach is to encourage simultaneously individual accuracy and diversity within the ensemble. Diversity is promoted through the feature extraction for each base classifier. Decision trees were chosen here because they are sensitive to rotation of the feature axes, hence the name "forest." Accuracy is sought by keeping all principal components and also using the whole data set to train each base classifier. Using WEKA, we examined the Rotation Forest ensemble on a random selection of 33 benchmark data sets from the UCI repository and compared it with Bagging, AdaBoost, and Random Forest. The results were favorable to Rotation Forest and prompted an investigation into diversity-accuracy landscape of the ensemble models. Diversity-error diagrams revealed that Rotation Forest ensembles construct individual classifiers which are more accurate than these in AdaBoost and Random Forest, and more diverse than these in Bagging, sometimes more accurate as well.},
author = {Rodr\'{\i}guez, Juan J. and Kuncheva, Ludmila I. and Alonso, Carlos J.},
doi = {10.1109/TPAMI.2006.211},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Rodr\'{\i}guez, Kuncheva, Alonso - 2006 - Rotation forest A New classifier ensemble method.pdf:pdf},
isbn = {0162-8828 (Print)},
issn = {01628828},
journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
keywords = {AdaBoost,Bagging,Classifier ensembles,Feature extraction,Kappa-error diagrams,PCA,Random forest},
number = {10},
pages = {1619--1630},
pmid = {16986543},
title = {{Rotation forest: A New classifier ensemble method}},
volume = {28},
year = {2006}
}
@article{Janitza2012,
abstract = {The Random Forest (RF) algorithm by Leo Breiman has become a standard data analysis tool in bioinformatics. It has shown excellent performance in settings where the number of variables is much larger than the number of observations, can cope with complex interaction structures as well as highly correlated variables and returns measures of variable importance. This paper synthesizes ten years of RF devel- opment with emphasis on applications to bioinformatics and compu- tational biology. Special attention is given to practical aspects such as the selection of parameters, available RF implementations, and im- portant pitfalls and biases of RF and its variable importance measures (VIMs). The paper surveys recent developments of the methodology relevant to bioinformatics as well as some representative examples of RF applications in this context and possible directions for future re- search.},
author = {Boulesteix, Anne-laure and Janitza, Silke and Kruppa, Jochen and K\"{o}nig, Inke R.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Boulesteix et al. - 2012 - Overview of Random Forest Methodology and Practical Guidance with Emphasis on Computational Biologyand Bioinf.pdf:pdf},
journal = {Technical Report - Institut f\"{u}r Statistik - University of Munich},
number = {129},
title = {{Overview of Random Forest Methodology and Practical Guidance with Emphasis on Computational Biologyand Bioinformatics}},
year = {2012}
}
@article{Gehrke1998,
abstract = {Classification of large datasets is an important data mining problem. Many classification algorithms have been proposed in the literature, but studies have shown that so far no algorithm uniformly outperforms all other algorithms in terms of quality. In this paper, we present a unifying framework for decision tree classifiers that separates the scalability aspects of algorithms for constructing a decision tree from the central features that determine the quality of the tree. This generic algorithm is easy to instantiate with specific algorithms from the literature (including C4.5, CART, CHAID, FACT, ID3 and extensions, SLIQ, Sprint and QUEST). In addition to its generality, in that it yields scalable versions of a wide range of classification algorithms, our approach also offers performance improvements of over a factor of five over the Sprint algorithm, the fastest scalable classification algorithm proposed previously. In contrast to Sprint, however, our generic algorithm requires a certain minimum amount of main memory, proportional to the set of distinct values in a column of the input relation. Given current main memory costs, this reuirement is readily met in most if not all workloads.},
author = {Gehrke, Johannes and Ramakrishnan, R and Ganti, V},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Gehrke, Ramakrishnan, Ganti - 1998 - RainForest—a framework for fast decision tree construction of large datasets.pdf:pdf},
journal = {24rd International Conference on Very Large Data Bases},
pages = {416--427},
title = {{RainForest—a framework for fast decision tree construction of large datasets}},
url = {http://www.springerlink.com/index/G652567687177764.pdf},
year = {1998}
}
@article{Friedman1999a,
author = {Friedman, Jerome H},
doi = {10.2307/2699986},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Friedman - 1999 - Greedy Function Approximation A Gradient Boosting Machine 1 Function estimation 2 Numerical optimization in function.pdf:pdf},
isbn = {0090-5364},
issn = {00905364},
journal = {North},
number = {3},
pages = {1--10},
title = {{Greedy Function Approximation : A Gradient Boosting Machine 1 Function estimation 2 Numerical optimization in function space}},
volume = {1},
year = {1999}
}
@article{Ghattas2000,
author = {Ghattas, B},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ghattas - 2000 - Importance des variables dans les m\'{e}thodes CART.pdf:pdf},
title = {{Importance des variables dans les m\'{e}thodes CART}},
url = {http://greqam.univ-mrs.fr/IMG/working\_papers/2000/00b04.pdf},
year = {2000}
}
@article{Culver2006a,
author = {Culver, Matt and Kun, Deng and Scott, Stephen},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Culver, Kun, Scott - 2006 - Active Learning to Maximize Area Under the ROC Curve.pdf:pdf},
isbn = {0769527019},
journal = {Proceedings of the Sixth International Conference on Data Mining},
pages = {149--158},
title = {{Active Learning to Maximize Area Under the ROC Curve}},
year = {2006}
}
@article{Pavlidis2012,
abstract = {Credit scoring methods for predicting creditworthiness have proven very effective in consumer finance. In light of the present financial crisis, such methods will become even more important. One of the outstanding issues in credit risk classification is population drift. This term refers to changes occurring in the population due to unexpected changes in economic conditions and other factors. In this paper, we propose a novel methodology for the classification of credit applications that has the potential to adapt to population drift as it occurs. This provides the opportunity to update the credit risk classifier as new labelled data arrives. Assorted experimental results suggest that the proposed method has the potential to yield significant performance improvement over standard approaches, without sacrificing the classifier's descriptive capabilities.},
author = {Pavlidis, N. G. and Tasoulis, Dimitrios K. and Adams, N. M. and Hand, David J.},
doi = {10.1057/jors.2012.15},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Pavlidis et al. - 2012 - Adaptive consumer credit classification.pdf:pdf},
issn = {0160-5682},
keywords = {HB Economic Theory},
number = {12},
pages = {1645--1654},
publisher = {Nature Publishing Group},
title = {{Adaptive consumer credit classification}},
url = {http://dx.doi.org/10.1057/jors.2012.15},
volume = {63},
year = {2012}
}
@article{CorinnaCortes2005,
abstract = {Inmany applications, good ranking is a highly desirable performance for a classifier. The criterion commonly used to measure the ranking quality of a classification algorithm is the area under the ROC curve (AUC). To report it properly, it is crucial to determine an interval of confidence for its value. This paper provides confidence intervals for the AUC based on a statistical and combinatorial analysis using only simple parameters such as the error rate and the number of positive and negative examples. The analysis is distribution-independent, it makes no assumption about the distribution of the scores of negative or positive examples. The results are of practical use and can be viewed as the equivalent for AUC of the standard confidence intervals given in the case of the error rate. They are comparedwith previous approaches in several standard classification tasks demonstrating the benefits of our analysis. 1},
author = {Cortes, Corinna and Mohri, Mehryar},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cortes, Mohri - 2005 - Confidence intervals for the area under the ROC curve.pdf:pdf},
journal = {Advances in Neural Information Processing Systems \ldots},
pages = {305--312},
title = {{Confidence intervals for the area under the ROC curve}},
url = {http://books.google.com/books?hl=en\&lr=\&id=etp-l5VrbHsC\&oi=fnd\&pg=PA305\&dq=Confidence+Intervals+for+the+Area+under+the+ROC+Curve\&ots=\_K6x0GtGxG\&sig=\_clX-1y-IV17gIDSI5c63gBewSg},
volume = {17},
year = {2005}
}
@article{Dupret2001,
abstract = {This paper presents a technical framework to assess the impact of re-sampling on the ability of a supervised learning to correctly learn a classification problem. We use the bootstrap expression of the prediction error to identify the optimal re-sampling proportions in binary classification experiments using artificial neural networks. Based on Bayes decision rule and the a priori distribution of the objective data, an estimate for the optimal re-sampling proportion is derived as well as upper and lower bounds for the exact optimal proportion. The analytical considerations to extend the present method to cross-validation and multiple classes are also illustrated. ?? 2001 Elsevier Science B.V.},
author = {Dupret, Georges and Koda, Masato},
doi = {10.1016/S0377-2217(00)00244-7},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Dupret, Koda - 2001 - Bootstrap re-sampling for unbalanced data in supervised learning.pdf:pdf},
issn = {03772217},
journal = {European Journal of Operational Research},
keywords = {Data mining,Decision support systems,Neural networks,Simulation},
number = {1},
pages = {141--156},
title = {{Bootstrap re-sampling for unbalanced data in supervised learning}},
volume = {134},
year = {2001}
}
@article{Shannon1957,
archivePrefix = {arXiv},
arxivId = {chao-dyn/9411012},
author = {Shannon, C.},
doi = {10.1145/584091.584093},
eprint = {9411012},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Shannon - 1948 - A mathematical theory of communication.pdf:pdf},
isbn = {1559-1662},
issn = {07246811},
journal = {Bell System Technology Journal},
pages = {379:423, 623--656},
pmid = {9230594},
primaryClass = {chao-dyn},
title = {{A mathematical theory of communication}},
volume = {27},
year = {1948}
}
@article{Freund2003,
abstract = {We study the problem of learning to accurately rank a set of objects by combining a given collection of ranking or preference functions. This problem of combining preferences arises in several applications, such as that of combining the results of different search engines, or the “collaborative-filtering” problem of ranking movies for a user based on the movie rankings provided by other users. In this work, we begin by presenting a formal framework for this general problem. We then describe and analyze an efficient algorithm called RankBoost for combining preferences based on the boosting approach to machine learning. We give theoretical results describing the algorithm’s behavior both on the training data, and on new test data not seen during training. We also describe an efficient implementation of the algorithm for a particular restricted but common case. We next discuss two experiments we carried out to assess the performance of RankBoost. In the first experiment, we used the algorithm to combine different web search strategies, each of which is a query expansion for a given domain. The second experiment is a collaborative-filtering task for making movie recommendations.},
author = {Freund, Yoav and Iyer, Raj and Schapire, Robert E and Singer, Yoram},
doi = {10.1162/jmlr.2003.4.6.933},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Freund et al. - 2003 - An Efficient Boosting Algorithm for Combining Preferences.pdf:pdf},
isbn = {1581134924},
issn = {15324435},
journal = {The Journal of Machine Learning Research},
pages = {933--969},
pmid = {345},
title = {{An Efficient Boosting Algorithm for Combining Preferences}},
volume = {4},
year = {2003}
}
@article{Herschtal2004,
author = {Herschtal, Alan and Raskutti, Bhavani},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Herschtal, Raskutti - 2004 - Optimising area under the ROC curve using gradient descent.pdf:pdf},
journal = {Proceedings of the twenty-first international \ldots},
title = {{Optimising area under the ROC curve using gradient descent}},
url = {http://dl.acm.org/citation.cfm?id=1015366},
year = {2004}
}
@article{Bernard2012,
abstract = {In this paper, we introduce a new Random Forest (RF) induction algorithm called Dynamic Random Forest (DRF) which is based on an adaptative tree induction procedure. The main idea is to guide the tree induction so that each tree will complement as much as possible the existing trees in the ensemble. This is done here through a resampling of the training data, inspired by boosting algorithms, and combined with other randomization processes used in traditional RF methods. The DRF algorithm shows a significant improvement in terms of accuracy compared to the standard static RF induction algorithm. © 2012 Elsevier B.V. All rights reserved.},
author = {Bernard, Simon and Adam, S\'{e}bastien and Heutte, Laurent},
doi = {10.1016/j.patrec.2012.04.003},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bernard, Adam, Heutte - 2012 - Dynamic Random Forests.pdf:pdf},
isbn = {0167-8655},
issn = {01678655},
journal = {Pattern Recognition Letters},
keywords = {Dynamic induction,Ensemble of classifiers,Random feature selection,Random forests},
number = {12},
pages = {1580--1586},
title = {{Dynamic Random Forests}},
volume = {33},
year = {2012}
}
@article{Khalilia2011,
abstract = {ABSTRACT:},
author = {Khalilia, Mohammed and Chakraborty, Sounak and Popescu, Mihail},
doi = {10.1186/1472-6947-11-51},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Khalilia, Chakraborty, Popescu - 2011 - Predicting disease risks from highly imbalanced data using random forest.pdf:pdf},
isbn = {1472-6947 (Electronic)$\backslash$r1472-6947 (Linking)},
issn = {1472-6947},
journal = {BMC medical informatics and decision making},
number = {1},
pages = {51},
pmid = {21801360},
publisher = {BioMed Central Ltd},
title = {{Predicting disease risks from highly imbalanced data using random forest.}},
url = {http://www.biomedcentral.com/1472-6947/11/51},
volume = {11},
year = {2011}
}
@article{Deng,
abstract = {Tree ensembles such as randomforests and boosted trees are accurate but dif- ficult to understand, debug and deploy. In this work, we provide the inTrees (interpretable trees) framework that extracts, measures, prunes and selects rules from a tree ensemble, and calculates frequent variable interactions. An rule-based learner, referred to as the simplified tree ensemble learner (STEL), can also be formed and used for future prediction. The inTrees framework can applied to both classification and regression problems, and is applicable to many types of tree ensembles, e.g., random forests, regularized random forests, and boosted trees. We implemented the inTrees algorithms in the “inTrees” R package. Keywords:},
archivePrefix = {arXiv},
arxivId = {arXiv:1408.5456v1},
author = {Deng, Houtao},
eprint = {arXiv:1408.5456v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Deng - Unknown - Interpreting Tree Ensembles with inTrees.pdf:pdf},
keywords = {decision tree,random forest,rule extraction,rule-based learner},
title = {{Interpreting Tree Ensembles with inTrees}}
}
@article{Chipman1998,
abstract = {In this article we put forward a Bayesian approach for finding classification and regression tree (CART) models. The two basic components of this approach consist of prior specification and stochastic search. The basic idea is to have the prior induce a posterior distribution that will guide the stochastic search toward more promising CART models. As the search proceeds, such models can then be selected with a variety of criteria, such as posterior probability, marginal likelihood, residual sum of squares or misclassification rates. Examples are used to illustrate the potential superiority of this approach over alternative methods.},
author = {Chipman, Hugh a. and George, Edward I. and McCulloch, Robert E.},
doi = {10.2307/2669832},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Chipman, George, McCulloch - 1998 - Bayesian CART Model Search.pdf:pdf},
issn = {0162-1459},
journal = {Journal of the American Statistical Association},
keywords = {binary trees,markov chain monte carlo,mixture,stochastic search to-,that will guide the},
number = {443},
pages = {935--948},
title = {{Bayesian CART Model Search}},
url = {http://www.jstor.org/stable/2669832$\backslash$nhttp://www.jstor.org.libproxy1.nus.edu.sg/stable/pdfplus/2669832.pdf?acceptTC=true},
volume = {93},
year = {1998}
}
@article{Yan2003,
abstract = {When the goal is to achieve the best correct classification rate, cross entropy and mean squared error are typical cost functions used to optimize classifier performance. However, for many real-world classification problems, the ROC curve is a more meaningful perfor- mance measure. We demonstrate that min- imizing cross entropy or mean squared error does not necessarily maximize the area un- der the ROC curve (AUC).We then consider alternative objective functions for training a classifier to maximize the AUC directly. We propose an objective function that is an ap- proximation to the Wilcoxon-Mann-Whitney statistic, which is equivalent to the AUC. The proposed objective function is differentiable, so gradient-based methods can be used to train the classifier. We apply the new objec- tive function to real-world customer behav- ior prediction problems for a wireless service provider and a cable service provider, and achieve reliable improvements in the ROC curve.},
author = {Yan, Lian and Dodier, Robert and Mozer, Michael C and Wolniewicz, Richard},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Yan et al. - 2003 - Optimizing Classifier Performance via an Approximation to the Wilcoxon-Mann-Whitney Statistic.pdf:pdf},
isbn = {1577351894},
journal = {Machine Learninginternational Workshop Then Conference},
number = {2},
pages = {848},
title = {{Optimizing Classifier Performance via an Approximation to the Wilcoxon-Mann-Whitney Statistic}},
url = {http://scholar.google.com/scholar?hl=en\&btnG=Search\&q=intitle:Optimizing+Classifier+Performance+via+an+Approximation+to+the+Wilcoxon-Mann-Whitney+Statistic\#0},
volume = {20},
year = {2003}
}
@article{Dahinden2009,
abstract = {Random Forests is a popular ensemble technique developed by Breiman (2001) which yields exceptional performance. These excellent results are achieved with little need to fine-tune parameters. The method is computationally effective, does not overfit, is robust to noise and can also be applied when the number of variables is much larger than the number of samples. We propose a slightly modified Random Forests scheme, with cross-validation as a means for tuning parameters and estimating error-rates. This simple and computationally very efficient approach was found to yield better predictive performance on the WCCI 2006 Performance Prediction Challenge datasets than many algorithms of much higher complexity},
author = {Dahinden, Corinne},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Dahinden - 2009 - An improved Random Forests approach with application to the performance prediction challenge datasets.pdf:pdf},
journal = {Hands on Pattern Recognition. Microtome},
keywords = {ensemble methods,random forests},
pages = {1--6},
title = {{An improved Random Forests approach with application to the performance prediction challenge datasets}},
url = {http://stat.ethz.ch/~dahinden/Paper/Bookchapter.pdf},
year = {2009}
}
@article{Blagus2010,
abstract = {The goal of class prediction studies is to develop rules to accurately predict the class membership of new samples. The rules are derived using the values of the variables available for each subject: the main characteristic of high-dimensional data is that the number of variables greatly exceeds the number of samples. Frequently the classifiers are developed using class-imbalanced data, i.e., data sets where the number of samples in each class is not equal. Standard classification methods used on class-imbalanced data often produce classifiers that do not accurately predict the minority class; the prediction is biased towards the majority class. In this paper we investigate if the high-dimensionality poses additional challenges when dealing with class-imbalanced prediction. We evaluate the performance of six types of classifiers on class-imbalanced data, using simulated data and a publicly available data set from a breast cancer gene-expression microarray study. We also investigate the effectiveness of some strategies that are available to overcome the effect of class imbalance.},
author = {Blagus, Rok and Lusa, Lara},
doi = {10.1186/1471-2105-11-523},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Blagus, Lusa - 2010 - Class prediction for high-dimensional class-imbalanced data.pdf:pdf},
issn = {1471-2105},
journal = {BMC bioinformatics},
number = {1},
pages = {523},
pmid = {20961420},
publisher = {BioMed Central Ltd},
title = {{Class prediction for high-dimensional class-imbalanced data.}},
url = {http://www.biomedcentral.com/1471-2105/11/523},
volume = {11},
year = {2010}
}
@article{Homescu2011,
abstract = {Two of the most important areas in computational finance: Greeks and, respectively, calibration, are based on efficient and accurate computation of a large number of sensitivities. This paper gives an overview of adjoint and automatic differentiation (AD), also known as algorithmic differentiation, techniques to calculate these sensitivities. When compared to finite difference approximation, this approach can potentially reduce the computational cost by several orders of magnitude, with sensitivities accurate up to machine precision. Examples and a literature survey are also provided.},
archivePrefix = {arXiv},
arxivId = {1107.1831},
author = {Homescu, Cristian},
eprint = {1107.1831},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Homescu - 2011 - Adjoints and Automatic (Algorithmic) Differentiation in Computational Finance.pdf:pdf},
pages = {23},
title = {{Adjoints and Automatic (Algorithmic) Differentiation in Computational Finance}},
url = {http://arxiv.org/abs/1107.1831},
year = {2011}
}
@unpublished{Yeh2014,
author = {Yeh, Shu-hao and Wang, Chuan-ju and Tsai, Ming-Feng},
booktitle = {Intermational Institute of Forecasting},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Yeh, Wang, Tsai - 2014 - Corporate Default Prediction via Deep Learning.pdf:pdf},
keywords = {deep learning,default prediction},
title = {{Corporate Default Prediction via Deep Learning}},
url = {http://forecasters.org/wp/wp-content/uploads/gravity\_forms/7-2a51b93047891f1ec3608bdbd77ca58d/2014/07/Yeh\_Shu-Hao\_ISF2014.pdf},
year = {2014}
}
@article{Wolpert1999,
abstract = {Bagging (Breiman, 1994a) is a technique that tries to improve a learning algorithm's performance by using bootstrap replicates of the training set (Efron \& Tibshirani, 1993, Efron, 1979). The computational requirements for estimating the resultant generalization error on a test set by means of cross-validation are often prohibitive, for leave-one-out cross-validation one needs to train the underlying algorithm on the order of m? times, where m is the size of the training set and ? is the number of replicates. This paper presents several techniques for estimating the generalization error of a bagged learning algorithm without invoking yet more training of the underlying learning algorithm (beyond that of the bagging itself), as is required by cross-validation-based estimation. These techniques all exploit the bias-variance decomposition (Geman, Bienenstock \& Doursat, 1992, Wolpert, 1996). The best of our estimators also exploits stacking (Wolpert, 1992). In a set of experiments reported here, it was found to be more accurate than both the alternative cross-validation-based estimator of the bagged algorithm's error and the cross-validation-based estimator of the underlying algorithm's error. This improvement was particularly pronounced for small test sets. This suggests a novel justification for using bagging—more accurate estimation of the generalization error than is possible without bagging.},
author = {Wolpert, David H. and Macready, William G.},
doi = {10.1023/A:1007519102914},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wolpert, Macready - 1999 - Efficient method to estimate Bagging's generalization error.pdf:pdf},
isbn = {0885-6125},
issn = {08856125},
journal = {Machine Learning},
keywords = {bagging,bootstrap,cross-validation,generalization error,stacking},
number = {1},
pages = {41--55},
title = {{Efficient method to estimate Bagging's generalization error}},
volume = {35},
year = {1999}
}
@article{Mylonakis2010,
abstract = {The paper attempts to determine whether there exists a relationship between the on-time payments of credit card owners of a Commercial Bank and their demographic characteristics (particular personal and family status). It evaluates the statistical technique of discriminant analysis on credit card customers’ data of a Greek Commercial Bank and examine whether it is possible to create a model evaluating the credibility of prospective credit-card customer. The sample includes personal data, as well as, payment consistency for 829 customers of the Greek Commercial Bank (X-BANK) of average size.The statistical analysis of the sample data included the identification of the relationship between the theoretical and empirical prices of the distributions of the bank customers’ specific variables and discriminant analysis. The results showed that establishing a model to evaluate the credibility of prospective bank card customers, using the technique of the linear discriminant analysis, is not possible. The findings prove interesting and useful for all bank managers. The paper contributes to the financial services literature by adding a further critical analysis into credit scoring systems established by several banking institutions.},
author = {Mylonakis, John and Diacogiannis, George},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Mylonakis, Diacogiannis - 2010 - Evaluating the likelihood of using linear discriminant analysis as a commercial bank card owners credit.pdf:pdf},
journal = {International business research},
keywords = {bank consumer credit risk,bank credit card,banking,credit scoring systems,financial risk},
number = {2},
pages = {P9},
title = {{Evaluating the likelihood of using linear discriminant analysis as a commercial bank card owners credit scoring model}},
url = {http://ccsenet.org/journal/index.php/ibr/article/view/5621$\backslash$nhttp://content.ebscohost.com.libezproxy.open.ac.uk/ContentServer.asp?T=P\&P=AN\&K=49187496\&S=R\&D=bth\&EbscoContent=dGJyMNHr7ESep7c4v+vlOLCmr0qeprNSsKe4TLSWxWXS\&ContentCustomer=dGJyMPGtt0qyprBQuczBa/},
volume = {3},
year = {2010}
}
@article{Joly,
abstract = {We adapt the idea of random projections applied to the out- put space, so as to enhance tree-based ensemble methods in the context of multi-label classification. We show how learning time complexity can be reduced without affecting computational complexity and accuracy of predictions.We also show that random output space projections may be used in order to reach different bias-variance tradeoffs, over a broad panel of benchmark problems, and that this may lead to improved accuracy while reducing significantly the computational burden of the learning stage.},
archivePrefix = {arXiv},
arxivId = {arXiv:1404.3581v4},
author = {Joly, Arnaud and Geurts, Pierre and Wehenkel, Louis},
eprint = {arXiv:1404.3581v4},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Joly, Geurts, Wehenkel - Unknown - Random forests with random projections of the output space for high dimensional multi-label classific.pdf:pdf},
title = {{Random forests with random projections of the output space for high dimensional multi-label classification}}
}
@book{Cherkassky2007,
author = {Cherkassky, Vladimir and Mulier, Filip},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cherkassky, Mulier - 2007 - Learning from Data.pdf:pdf},
isbn = {978-0-471-68182-3},
title = {{Learning from Data}},
year = {2007}
}
@article{Ren2015,
abstract = {Random forest is well known as one of the best learning methods. In spite of its great success, it also has certain drawbacks: the heuristic learning rule does not effectively minimize the global training loss; the model size is usually too large for many real applications. To address the issues, we propose two techniques, global refinement and global pruning, to improve a pre-trained random forest. The pro- posed global refinement jointly relearns the leaf nodes of all trees under a global objective function so that the comple- mentary information between multiple trees is well exploit- ed. In this way, the fitting power of the forest is significantly enhanced. The global pruning is developed to reduce the model size as well as the over-fitting risk. The refined model has better performance and smaller storage cost, as verified in extensive experiments.},
author = {Ren, Shaoqing and Cao, Xudong and Wei, Yichen and Sun, Jian},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ren et al. - 2015 - Global Refinement of Random Forest.pdf:pdf},
journal = {CVPR},
title = {{Global Refinement of Random Forest}},
year = {2015}
}
@article{Joshi2009,
author = {Joshi, M. S. and Yang, C.},
doi = {10.3233/AF-2011-002},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Joshi, Yang - 2009 - Efficient Greek estimation in generic market models.pdf:pdf},
issn = {21576203},
journal = {SSRN},
keywords = {adjoint method,and phrases,computational order,delta,market model,monte,vega},
pages = {1--27},
title = {{Efficient Greek estimation in generic market models}},
url = {http://papers.ssrn.com/sol3/papers.cfm?abstract\_id=1437847},
year = {2009}
}
@inproceedings{Saffari2009,
abstract = {Random Forests (RFs) are frequently used in many computer vision and machine learning applications. Their popularity is mainly driven by their high computational efficiency during both training and evaluation while achieving state-of-the-art results. However, in most applications RFs are used off-line. This limits their usability for many practical problems, for instance, when training data arrives sequentially or the underlying distribution is continuously changing. In this paper, we propose a novel on-line random forest algorithm. We combine ideas from on-line bagging, extremely randomized forests and propose an on-line decision tree growing procedure. Additionally, we add a temporal weighting scheme for adaptively discarding some trees based on their out-of-bag-error in given time intervals and consequently growing of new trees. The experiments on common machine learning data sets show that our algorithm converges to the performance of the off-line RF. Additionally, we conduct experiments for visual tracking, where we demonstrate real-time state-of-the-art performance on well-known scenarios and show good performance in case of occlusions and appearance changes where we outperform trackers based on on-line boosting. Finally, we demonstrate the usability of on-line RFs on the task of interactive real-time segmentation.},
author = {Saffari, Amir and Leistner, Christian and Santner, Jakob and Godec, Martin and Bischof, Horst},
booktitle = {2009 IEEE 12th International Conference on Computer Vision Workshops, ICCV Workshops 2009},
doi = {10.1109/ICCVW.2009.5457447},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Saffari et al. - 2009 - On-line random forests.pdf:pdf},
isbn = {9781424444427},
issn = {08856125},
pages = {1393--1400},
pmid = {20142443},
title = {{On-line random forests}},
year = {2009}
}
@article{Denil2013,
abstract = {As a testament to their success, the theory of random forests has long been outpaced by their application in practice. In this paper, we take a step towards narrowing this gap by providing a consistency result for online random forests.},
archivePrefix = {arXiv},
arxivId = {arXiv:1302.4853v2},
author = {Denil, Misha and Matheson, David and {De Freitas}, Nando},
eprint = {arXiv:1302.4853v2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Denil, Matheson, De Freitas - 2013 - Consistency of Online Random Forests.pdf:pdf},
title = {{Consistency of Online Random Forests}},
year = {2013}
}
@article{Ehm2010,
author = {Ehm, Werner and Gneiting, Tilmann},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ehm, Gneiting - 2010 - Local Proper Scoring Rules Local Proper Scoring Rules.pdf:pdf},
number = {551},
pages = {695--709},
title = {{Local Proper Scoring Rules Local Proper Scoring Rules}},
year = {2010}
}
@article{Liu,
abstract = {Mining imbalanced data has recently received increasing attention due to its challenge and wide applications in the real world. Most of the existing work focuses on numerical data by manipulating the data structure which essentially changes the data characteristics or developing new distance or similarity measures which are designed for data with the so-called IID assumption, namely data is independent and identically distributed. This is not consistent with the real- life data and business needs, which request to fully respect the data structure and coupling relationships embedded in data objects, features and feature values. In this paper, we propose a novel coupled fuzzy similarity-based classification approach to cater for the difference between classes by a fuzzy membership and the couplings by coupled object similarity, and incorporate them into the most popular classifier: kNN to form a coupled fuzzy kNN (ie. CF-kNN). We test the approach on 14 categorical data sets compared to several kNN variants and classic classifiers including C4.5 and NaiveBayes. The experimental results show that CF-kNN outperforms the baselines, and those classifiers incorporated with the proposed coupled fuzzy similarity perform better than their original editions. I.},
author = {Liu, Chunming and Cao, Longbing and Yu, Philip S},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Liu, Cao, Yu - Unknown - Coupled Fuzzy k-Nearest Neighbors Classification of Imbalanced Non-IID Categorical Data.pdf:pdf},
title = {{Coupled Fuzzy k-Nearest Neighbors Classification of Imbalanced Non-IID Categorical Data}}
}
@article{Scornet2012,
abstract = {Random forests are ensemble methods which grow trees as base learners and combine their predictions by averaging. Random forests are known for their good practical performance, particularly in high dimensional set- tings. On the theoretical side, several studies highlight the potentially fruitful connection between random forests and kernel methods. In this paper, we work out in full details this connection. In particular, we show that by slightly modifying their definition, random forests can be rewrit- ten as kernel methods (called KeRF for Kernel based on Random Forests) which are more interpretable and easier to analyze. Explicit expressions of KeRF estimates for some specific random forest models are given, together with upper bounds on their rate of consistency. We also show empirically that KeRF estimates compare favourably to random forest estimates.},
archivePrefix = {arXiv},
arxivId = {1502.03836v1},
author = {Scornet, Erwan},
eprint = {1502.03836v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Scornet - 2012 - Random forests and kernel methods.pdf:pdf;:Users/guillaume/Dropbox/Mendeley Desktop/Scornet - 2012 - Random forests and kernel methods(2).pdf:pdf},
journal = {HAL},
keywords = {Random forests,consistency,kernel methods.,randomization,rate of con- sistency},
pages = {1--30},
title = {{Random forests and kernel methods}},
year = {2012}
}
@article{Bovier2014,
abstract = {We extend the results of Arguin et al [4] and A¨ ekon et al [1] on the conver- ıd´ gence of the extremal process of branching Brownian motion by adding an extra dimension that encodes the ”location” of the particle in the underlying Galton-Watson tree. We show that the limit is a cluster point process on R+ × R where each cluster is the atom of a Poisson point process on R+ × R with a random intensity measure Z(dz) × Ce− √2x, where the random measure is explicitly constructed from the derivative martingale. This work is motivated by an analogous conjecture for the Gaussian free field by Biskup and Louidor},
archivePrefix = {arXiv},
arxivId = {arXiv:1412.5975v1},
author = {Bovier, Anton and Hartung, Lisa},
eprint = {arXiv:1412.5975v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bovier, Hartung - 2014 - Extended Convergence of the Extremal Process of Branching Brownian Motion.pdf:pdf},
keywords = {and phrases,branching brownian motion,cluster,extremal processes,gaussian processes,multiplicative chaos,processes},
pages = {1--16},
title = {{Extended Convergence of the Extremal Process of Branching Brownian Motion}},
year = {2014}
}
@article{Learning2005,
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Unknown - 2005 - Proceedings of the Workshop on Feature Selection for Data Mining Interfacing Machine Learning and Statistics.pdf:pdf},
journal = {2005 SIAM International Conference on Data Mining},
title = {{Proceedings of the Workshop on Feature Selection for Data Mining : Interfacing Machine Learning and Statistics}},
year = {2005}
}
@article{Clemencon2006,
abstract = {The problem of ranking/ordering instances, instead of simply classifying them, has recently gained much attention in machine learning. In this paper we formulate the ranking problem in a rigorous statistical framework. The goal is to learn a ranking rule for deciding, among two instances, which one is "better," with minimum ranking risk. Since the natural estimates of the risk are of the form of a U-statistic, results of the theory of U-processes are required for investigating the consistency of empirical risk minimizers. We establish in particular a tail inequality for degenerate U-processes, and apply it for showing that fast rates of convergence may be achieved under specific noise assumptions, just like in classification. Convex risk minimization methods are also studied.},
archivePrefix = {arXiv},
arxivId = {math/0603123},
author = {Cl\'{e}men\c{c}on, St\'{e}phan and Lugosi, G\'{a}bor and Vayatis, Nicolas},
eprint = {0603123},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cl\'{e}men\c{c}on, Lugosi, Vayatis - 2006 - Ranking and empirical minimization of U-statistics.pdf:pdf},
primaryClass = {math},
title = {{Ranking and empirical minimization of U-statistics}},
url = {http://arxiv.org/abs/math/0603123},
year = {2006}
}
@article{Batista2004,
abstract = {There are several aspects that might influence the performance achieved by existing learning systems. It has been reported that one of these aspects is related to class imbalance in which examples in training data belonging to one class heavily outnumber the examples in the other class. In this situation, which is found in real world data describing an infrequent but important event, the learning system may have difficulties to learn the concept related to the minority class. In this work we perform a broad experimental evaluation involving ten methods, three of them proposed by the authors, to deal with the class imbalance problem in thirteen UCI data sets. Our experiments provide evidence that class imbalance does not systematically hinder the performance of learning systems. In fact, the problem seems to be related to learning with too few minority class examples in the presence of other complicating factors, such as class overlapping. Two of our proposed methods deal with these conditions directly, allying a known over-sampling method with data cleaning methods in order to produce better-defined class clusters. Our comparative experiments show that, in general, over-sampling methods provide more accurate results than under-sampling methods considering the area under the ROC curve (AUC). This result seems to contradict results previously published in the literature. Two of our proposed methods, Smote + Tomek and Smote + ENN, presented very good results for data sets with a small number of positive examples. Moreover, Random over-sampling, a very simple over-sampling method, is very competitive to more complex over-sampling methods. Since the over-sampling methods provided very good performance results, we also measured the syntactic complexity of the decision trees induced from over-sampled data. Our results show that these trees are usually more complex then the ones induced from original data. Random over-sampling usually produced the smallest increase in the mean number of induced rules and Smote + ENN the smallest increase in the mean number of conditions per rule, when compared among the investigated over-sampling methods.},
author = {Batista, Gustavo E. A. P. A and Prati, Ronaldo C and Monard, Maria Carolina},
doi = {10.1145/1007730.1007735},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Batista, Prati, Monard - 2004 - A study of the behavior of several methods for balancing machine learning training data.pdf:pdf},
isbn = {1931-0145},
issn = {19310145},
journal = {ACM SIGKDD Explorations Newsletter},
number = {1},
pages = {20},
title = {{A study of the behavior of several methods for balancing machine learning training data}},
volume = {6},
year = {2004}
}
@article{Hernandez-Lobato2009,
author = {Hern\'{a}ndez-Lobato, Daniel},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hern\'{a}ndez-Lobato - 2009 - Prediction Based on Averages over Automatically Induced Learners Ensemble Methods and Bayesian Techniques.pdf:pdf},
number = {November},
pages = {149--164},
title = {{Prediction Based on Averages over Automatically Induced Learners : Ensemble Methods and Bayesian Techniques}},
year = {2009}
}
@article{Giles2005,
abstract = {This paper presents an adjoint method to accelerate the calculation of Greeks by Monte Carlo simulation. The method calculates price sensitivities along each path; but in contrast to a forward pathwise calculation, it works backward recursively using adjoint variables. Along each path, the forward and adjoint implementations produce the same values, but the adjoint method rearranges the calculations to generate potential computational savings. The adjoint method outperforms a forward implementation in calculating the sensitivities of a small number of outputs to a large number of inputs. This applies, for example, in estimating the sensitivities of an interest rate derivatives book to multiple points along an initial forward curve or the sensitivities of an equity derivatives book to multiple points on a volatility surface. We illustrate the application of the method in the setting of the LIBOR market model. Numerical results confirm that the computational advantage of the adjoint method grows in proportion to the number of initial forward rates.},
author = {Giles, M. and Glasserman, P.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Giles, Glasserman - 2005 - Smoking Adjoints fast evaluation of Greeks in Monte Carlo calculations.pdf:pdf},
keywords = {Numerical analysis},
number = {05},
pages = {1--15},
title = {{Smoking Adjoints: fast evaluation of Greeks in Monte Carlo calculations}},
url = {http://eprints.maths.ox.ac.uk/1138/},
year = {2005}
}
@article{Simon,
author = {Simon, Richard},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Simon - Unknown - Resampling Strategies for Model Assessment and Selection.pdf:pdf},
journal = {Cancer},
title = {{Resampling Strategies for Model Assessment and Selection}}
}
@book{Shalev-Shwartz2014,
author = {Shalev-Shwartz, S and Ben-David, S},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Shalev-Shwartz, Ben-David - 2014 - Understanding Machine Learning From Theory to Algorithms.pdf:pdf},
isbn = {9781107057135},
title = {{Understanding Machine Learning: From Theory to Algorithms}},
url = {http://scholar.google.com/scholar?hl=en\&btnG=Search\&q=intitle:Understanding+Machine+Learning:+From+Theory+to+Algorithms\#0},
year = {2014}
}
@article{Cour2011,
abstract = {We address the problem of partially-labeled multiclass classification, where instead of a single label per instance, the algorithm is given a candidate set of labels, only one of which is correct. Our setting is motivated by a common scenario in many image and video collections, where only partial access to labels is available. The goal is to learn a classifier that can disambiguate the partially-labeled training instances, and generalize to unseen data. We define an intuitive property of the data distribution that sharply characterizes the ability to learn in this setting and show that effective learning is possible even when all the data is only partially labeled. Exploiting this property of the data, we propose a convex learning formulation based on minimization of a loss function appropriate for the partial label setting. We analyze the conditions under which our loss function is asymptotically consistent, as well as its generalization and transductive performance. We apply our framework to identifying faces culled from web news sources and to naming characters in TV series and movies; in particular, we annotated and experimented on a very large video data set and achieve 6\% error for character naming on 16 episodes of the TV series Lost.},
author = {Cour, Timothee and Sapp, Benjamin},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cour, Sapp - 2011 - Learning from Partial Labels.pdf:pdf},
isbn = {1532-4435},
issn = {15324435},
journal = {Journal of Machine Learning Research},
keywords = {bounds,convex learning,generalization,multiclass classification,names faces,weakly supervised learning},
number = {2},
pages = {1501--1536},
title = {{Learning from Partial Labels}},
url = {http://jmlr.csail.mit.edu/papers/v12/cour11a.html},
volume = {12},
year = {2011}
}
@article{Lumijarvi2004,
abstract = {Proximity functions evaluate distances or similarities between objects. Unlike the Euclidean distance, heterogeneous proximity functions process variables differently according to their scale. The correct evaluation of nominal variables, whose values are unordered, is especially important. We compared five heterogeneous functions with the Euclidean distance to study whether functions sensitive to scale are better than a function assuming the same scale. In addition, we were interested of the relative performance of the five heterogeneous functions. The performance of the functions was measured with a nearest neighbor classifier that was applied to 12 medical data sets characterized with different scales. Unexpectedly, the performance of heterogeneous functions did not differ significantly from that of the Euclidean distance. As expected, significant differences between the Heterogeneous Value Difference Metric (HVDM) and the four value-matching-based heterogeneous functions favored HVDM. Additional research is needed to explain why heterogeneous functions did not outperform the Euclidean distance.},
author = {Lumij\"{a}rvi, Janne and Laurikkala, Jorma and Juhola, Martti},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Lumij\"{a}rvi, Laurikkala, Juhola - 2004 - A comparison of different heterogeneous proximity functions and Euclidean distance.pdf:pdf},
issn = {0926-9630},
journal = {Studies in health technology and informatics},
number = {Pt 2},
pages = {1362--1366},
pmid = {15361037},
title = {{A comparison of different heterogeneous proximity functions and Euclidean distance.}},
volume = {107},
year = {2004}
}
@article{Finlay2010,
abstract = {In consumer credit markets lending decisions are usually represented as a set of classification problems. The objective is to predict the likelihood of customers ending up in one of a finite number of states, such as good/bad payer, responder/non-responder and transactor/non-transactor. Decision rules are then applied on the basis of the resulting model estimates. However, this represents a misspecification of the true objectives of commercial lenders, which are better described in terms of continuous financial measures such as bad debt, revenue and profit contribution. In this paper, an empirical study is undertaken to compare predictive models of continuous financial behaviour with binary models of customer default. The results show models of continuous financial behaviour to outperform classification approaches. They also demonstrate that scoring functions developed to specifically optimize profit contribution, using genetic algorithms, outperform scoring functions derived from optimizing more general functions such as sum of squared error. ?? 2009 Elsevier B.V. All rights reserved.},
author = {Finlay, Steven},
doi = {10.1016/j.ejor.2009.05.025},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Finlay - 2010 - Credit scoring for profitability objectives.pdf:pdf},
issn = {03772217},
journal = {European Journal of Operational Research},
keywords = {Credit scoring,Genetic algorithms,OR in banking,Profitability},
number = {2},
pages = {528--537},
publisher = {Elsevier B.V.},
title = {{Credit scoring for profitability objectives}},
url = {http://dx.doi.org/10.1016/j.ejor.2009.05.025},
volume = {202},
year = {2010}
}
@article{Lamnisos2013,
archivePrefix = {arXiv},
arxivId = {arXiv:1306.6028v1},
author = {Lamnisos, Demetris and Griffin, Jim E and Steel, Mark F J},
eprint = {arXiv:1306.6028v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Lamnisos, Griffin, Steel - 2013 - Adaptive MC 3 and Gibbs Algorithms for Bayesian Model Averaging in Linear Regression Models.pdf:pdf},
keywords = {adaptive mcmc,gibbs sampler,model uncertainty,variable selection},
number = {13},
pages = {1--25},
title = {{Adaptive MC 3 and Gibbs Algorithms for Bayesian Model Averaging in Linear Regression Models}},
year = {2013}
}
@article{Mercer,
author = {Mercer, J.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Mercer - 1908 - Functions of Positive and Negative Type and their Connection with the Theory of Integral Equations.pdf:pdf},
journal = {Philosophical Transactions of the Royal Society of London},
title = {{Functions of Positive and Negative Type and their Connection with the Theory of Integral Equations}},
volume = {Series A, },
year = {1908}
}
@article{Wolpert1996,
abstract = {This is the first of two papers that use off-training set (OTS) error to investigate the assumption-free relationship between learning algorithms. This first paper discusses the senses in which there are no a priori distinctions between learning algorithms. (The second paper discusses the senses in which there are such distinctions.) In this first paper it is shown, loosely speaking, that for any two algorithms A and B, there are “as many” targets (or priors over targets) for which A has lower expected OTS error than B as vice versa, for loss functions like zero-one loss. In particular, this is true if A is cross-validation and B is “anti-cross-validation” (choose the learning algorithm with largest cross-validation error). This paper ends with a discussion of the implications of these results for computational learning theory. It is shown that one cannot say: if empirical misclassification rate is low, the Vapnik-Chervonenkis dimension of your generalizer is small, and the training set is large, then with high probability your OTS error is small. Other implications for “membership queries” algorithms and “punting” algorithms are also discussed.},
author = {Wolpert, David H.},
doi = {10.1162/neco.1996.8.7.1391},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wolpert - 1996 - The Existence of A Priori Distinctions Between Learning Algorithms.pdf:pdf},
isbn = {0899-7667},
issn = {0899-7667},
journal = {Neural Computation},
number = {7},
pages = {1391--1420},
title = {{The Existence of A Priori Distinctions Between Learning Algorithms}},
volume = {8},
year = {1996}
}
@article{Zenobi2001,
abstract = {It is well known that ensembles of predictors produce better accuracy than a single predictor provided there is diversity in the ensemble. This diversity manifests itself as disagreement or ambiguity among the ensemble members. In this paper we focus on ensembles of classifiers based on different feature subsets and we present a process for producing such ensembles that emphasizes diversity (ambiguity) in the ensemble members. This emphasis on diversity produces ensembles with low generalization errors from ensemble members with comparatively high generalization error. We compare this with ensembles produced focusing only on the error of the ensemble members (without regard to overall diversity) and find that the ensembles based on ambiguity have lower generalization error. Further, we find that the ensemble members produced focusing on ambiguity have less features on average that those based on error only. We suggest that this indicates that these ensemble members are local learners. },
author = {Zenobi, Gabriele and Cunningham, P\'{a}draig},
doi = {10.1007/3-540-44795-4\_49},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Zenobi, Cunningham - 2001 - Using Diversity in Preparing Ensembles of Classifiers Based on Different Feature Subsets to Minimize General.pdf:pdf},
isbn = {978-3-540-42536-6},
journal = {Machine Learning: ECML 2001},
number = {1995},
pages = {576--587},
title = {{Using Diversity in Preparing Ensembles of Classifiers Based on Different Feature Subsets to Minimize Generalization Error}},
url = {http://www.springerlink.com/index/10.1007/3-540-44795-4},
volume = {2167},
year = {2001}
}
@phdthesis{Timofeev2004,
abstract = {This master thesis is devoted to Classification and Regression Trees (CART). CART is classification method which uses historical data to construct decision trees. Depending on available information about the dataset, classification tree or regression tree can be constructed. Constructed tree can be then used for classification of new observations. The first part of the thesis describes fundamental principles of tree construction, dif- ferent splitting algorithms and pruning procedures. Second part of the paper answers the questions why should we use or should not use the CART method. Advantages and weaknesses of the method are discussed and tested in detail. In the last part, CART is applied to real data, using the statistical software XploRe. Here different statistical macros (quantlets), graphical and plotting tools are presented.},
author = {Timofeev, Roman},
booktitle = {Journal of neurosurgery},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Timofeev - 2004 - Classification and regression trees (CART) Theory and Applications.pdf:pdf},
issn = {0022-3085},
number = {5},
pmid = {7714600},
title = {{Classification and regression trees (CART) Theory and Applications}},
volume = {82},
year = {2004}
}
@article{Theorem,
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Unknown - Unknown - Markov Chains Ergodic Theorem.pdf:pdf},
pages = {1--3},
title = {{Markov Chains : Ergodic Theorem}}
}
@article{Zhang2010,
abstract = {Random forests have emerged as one of the most commonly used nonparametric statistical methods in many scientific areas, particularly in analysis of high throughput genomic data. A general practice in using random forests is to generate a sufficiently large number of trees, although it is subjective as to how large is sufficient. Furthermore, random forests are viewed as “black-box” because of its sheer size. In this work, we address a fundamental issue in the use of random forests: how large does a random forest have to be? To this end, we propose a specific method to find a sub-forest (e.g., in a single digit number of trees) that can achieve the prediction accuracy of a large random forest (in the order of thousands of trees). We tested it on extensive simulation studies and a real study on prognosis of breast cancer. The results show that such sub-forests usually exist and most of them are very small, suggesting they are actually the “representatives” of the whole random forests. We conclude that the sub-forests are indeed the core of a random forest. Thus it is not necessary to use the whole forest for satisfying prediction performance. Also, by reducing the size of a random forest to a manageable size, the random forest is no longer a black-box.},
author = {Zhang, Heping and Wang, Minghui},
doi = {10.1016/j.bbi.2008.05.010},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Zhang, Wang - 2010 - Search for the smallest random forest.pdf:pdf},
issn = {1938-7989},
journal = {Statistics and its interface},
keywords = {and phrases,classification,random forest,smallest forest},
number = {3},
pages = {381--381},
pmid = {20165560},
title = {{Search for the smallest random forest}},
url = {http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2822360/},
volume = {2},
year = {2009}
}
@article{Gall2005,
abstract = {We discuss several connections between discrete and continuous random trees. In the discrete setting, we focus on Galton-Watson trees under various conditionings. In particular, we present a simple approach to Aldous' theorem giving the convergence in distribution of the contour process of conditioned Galton-Watson trees towards the normalized Brownian excursion. We also briefly discuss applications to combinatorial trees. In the continuous setting, we use the formalism of real trees, which yields an elegant formulation of the convergence of rescaled discrete trees towards continuous objects. We explain the coding of real trees by functions, which is a continuous version of the well-known coding of discrete trees by Dyck paths. We pay special attention to random real trees coded by Brownian excursions, and in a particular we provide a simple derivation of the marginal distributions of the CRT. The last section is an introduction to the theory of the Brownian snake, which combines the genealogical structure of random real trees with independent spatial motions. We introduce exit measures for the Brownian snake and we present some applications to a class of semilinear partial differential equations.},
archivePrefix = {arXiv},
arxivId = {math/0511515},
author = {Gall, Jean-Francois Le},
doi = {10.1214/154957805100000140},
eprint = {0511515},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Gall - 2005 - Random trees and applications.pdf:pdf},
issn = {1549-5787},
keywords = {and phrases,brownian excursion,brownian motion,brownian snake,coding of trees,conditioned tree,contour process,crt,exit measure,partial differential equation,random tree,real tree,received september 2005},
pages = {245--311},
primaryClass = {math},
title = {{Random trees and applications}},
url = {http://arxiv.org/abs/math/0511515},
volume = {2},
year = {2005}
}
@article{Clemencon2009a,
author = {Clemencon, Stephan and Depecker, Marine and Vayatis, Nicolas},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Clemencon, Depecker, Vayatis - 2009 - AUC optimization and the two-sample problem.pdf:pdf},
isbn = {9781615679119},
journal = {Advances in Neural Information Processing Systems},
pages = {1--9},
title = {{AUC optimization and the two-sample problem}},
url = {http://papers.nips.cc/paper/3838-auc-optimization-and-the-two-sample-problem},
year = {2009}
}
@article{Murthy1994,
abstract = {This article describes a new system for induction of oblique decision trees. This system, OC1, combines deterministic hill-climbing with two forms of randomization to find a good oblique split (in the form of a hyperplane) at each node of a decision tree. Oblique decision tree methods are tuned especially for domains in which the attributes are numeric, although they can be adapted to symbolic or mixed symbolic/numeric attributes. We present extensive empirical studies, using both real and artificial data, that analyze OC1's ability to construct oblique trees that are smaller and more accurate than their axis-parallel counterparts. We also examine the benefits of randomization for the construction of oblique decision trees.},
archivePrefix = {arXiv},
arxivId = {cs/9408103},
author = {Murthy, Sreerama K. and Kasif, Simon and Salzberg, Steven},
doi = {10.1.1.44.6304},
eprint = {9408103},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Murthy, Kasif, Salzberg - 1994 - A System for Induction of Oblique Decision Trees.pdf:pdf},
pages = {1--32},
primaryClass = {cs},
title = {{A System for Induction of Oblique Decision Trees}},
url = {http://arxiv.org/abs/cs/9408103},
volume = {2},
year = {1994}
}
@book{Trevor,
author = {Hastie, Trevor and Tibshirani, Robert and Friedman, Jerome},
edition = {3},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hastie, Tibshirani, Friedman - 2008 - The Elements of Statistical Learning.pdf:pdf},
title = {{The Elements of Statistical Learning}},
url = {http://statweb.stanford.edu/~tibs/ElemStatLearn/printings/ESLII\_print10.pdf},
year = {2008}
}
@article{Breiman2001,
abstract = {Random forests are a combination of tree predictors such that each tree depends on the values of a random vector sampled independently and with the same distribution for all trees in the forest. The generalization error for forests converges a.s. to a limit as the number of trees in the forest becomes large. The generalization error of a forest of tree classifiers depends on the strength of the individual trees in the forest and the correlation between them. Using a random selection of features to split each node yields error rates that compare favorably to Adaboost (Y. Freund \& R. Schapire, Machine Learning: Proceedings of the Thirteenth International conference, ***, 148–156), but are more robust with respect to noise. Internal estimates monitor error, strength, and correlation and these are used to show the response to increasing the number of features used in the splitting. Internal estimates are also used to measure variable importance. These ideas are also applicable to regression.},
archivePrefix = {arXiv},
arxivId = {http://dx.doi.org/10.1023\%2FA\%3A1010933404324},
author = {Breiman, Leo},
doi = {10.1023/A:1010933404324},
eprint = {/dx.doi.org/10.1023\%2FA\%3A1010933404324},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Breiman - 2001 - Random forests.pdf:pdf},
isbn = {9781424444427},
issn = {08856125},
journal = {Machine Learning},
keywords = {Classification,Ensemble,Regression},
number = {1},
pages = {5--32},
pmid = {20142443},
primaryClass = {http:},
title = {{Random forests}},
volume = {45},
year = {2001}
}
@article{Davis2006,
abstract = {Receiver Operator Characteristic (ROC) curves are commonly used to present results for binary decision problems in machine learning. However, when dealing with highly skewed datasets, Precision-Recall (PR) curves give a more informative picture of an algorithm's performance. We show that a deep connection exists between ROC space and PR space, such that a curve dominates in ROC space if and only if it dominates in PR space. A corollary is the notion of an achievable PR curve, which has properties much like the convex hull in ROC space; we show an efficient algorithm for computing this curve. Finally, we also note differences in the two types of curves are significant for algorithm design. For example, in PR space it is incorrect to linearly interpolate between points. Furthermore, algorithms that optimize the area under the ROC curve are not guaranteed to optimize the area under the PR curve.},
author = {Davis, Jesse and Goadrich, Mark},
doi = {10.1145/1143844.1143874},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Davis, Goadrich - 2006 - The Relationship Between Precision-Recall and ROC Curves.pdf:pdf},
isbn = {1595933832},
issn = {14710080},
journal = {Proceedings of the 23rd International Conference on Machine learning -- ICML'06},
pages = {233--240},
pmid = {19165215},
title = {{The Relationship Between Precision-Recall and ROC Curves}},
url = {http://portal.acm.org/citation.cfm?doid=1143844.1143874},
year = {2006}
}
@article{Feurer2015,
author = {Feurer, Matthias and Klein, Aaron and Eggensperger, Katharina and Springenberg, Jost Tobias and Blum, Manuel and Hutter, Frank},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Feurer et al. - 2015 - Methods for Improving Bayesian Optimization for AutoML.pdf:pdf},
journal = {ICML 2015 AutoML Workshop},
keywords = {automated machine learning,bayesian optimization,ensemble construction},
title = {{Methods for Improving Bayesian Optimization for AutoML}},
year = {2015}
}
@article{Scornet2014,
abstract = {The last decade has witnessed a growing interest in random forest models which are recognized to exhibit good practical performance, especially in high-dimensional settings. On the theoretical side, however, their predictive power remains largely unexplained, thereby creating a gap between theory and practice. The aim of this paper is twofold. Firstly, we provide theoretical guarantees to link finite forests used in practice (with a finite number M of trees) to their asymptotic counterparts. Using empirical process theory, we prove a uniform central limit theorem for a large class of random forest estimates, which holds in particular for Breiman's original forests. Secondly, we show that infinite forest consistency implies finite forest consistency and thus, we state the consistency of several infinite forests. In particular, we prove that q quantile forests---close in spirit to Breiman's forests but easier to study---are able to combine inconsistent trees to obtain a final consistent prediction, thus highlighting the benefits of random forests compared to single trees.},
archivePrefix = {arXiv},
arxivId = {1409.2090},
author = {Scornet, Erwan},
eprint = {1409.2090},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Scornet - 2014 - On the asymptotics of random forests.pdf:pdf},
keywords = {Random forests,central limit theorem,consistency,empirical process,number of trees,q-quantile,randomization},
title = {{On the asymptotics of random forests}},
url = {http://arxiv.org/abs/1409.2090},
year = {2014}
}
@article{Miao2010,
abstract = {We compared two basic ensemble methods, namely random forest and Adaboost tree for the classification of ecosystems in Clark County, Nevada, USA through multitemporal multisource LANDSAT TM/ETM+ images and terrain-related GIS data layers. Random forest generates decision trees by randomly selecting a limited number features from all available features for node splitting, and each tree cast a vote for the final decision. On the other hand, Adaboost tree is an iterative approach to improve the performance of a weak classifier by assigning weights to training samples, and incorrectly classified training samples will gain a larger weight in the process. We discuss the properties of these two tree-based ensemble methods and compare their classification performances in ecosystem classification. The results show that Adaboost tree can provide higher classification accuracy than random forest in multitemporal multisource dataset, while the latter could be more efficient in computation.},
author = {Miao, Xin and Heaton, Jill S.},
doi = {10.1109/GEOINFORMATICS.2010.5567504},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Miao, Heaton - 2010 - A comparison of random forest and Adaboost tree in ecosystem classification in east Mojave Desert.pdf:pdf},
isbn = {9781424473021},
journal = {2010 18th International Conference on Geoinformatics, Geoinformatics 2010},
keywords = {Adaboost tree,Ecosystem,Random forest},
title = {{A comparison of random forest and Adaboost tree in ecosystem classification in east Mojave Desert}},
year = {2010}
}
@article{Journal2001,
abstract = {The consumer credit market is experiencing unprecedented change, increased competition, and new challenges. To cope with these developments, increasingly sophisticated mathematical and statistical tools are being used. Such tools are used to identify good and bad risks, to monitor customer performance, to characterize different behaviour patterns, and in a wide variety of other ways, at both individual and portfolio level. Examples of such applications and of the modern statistical tools developed to model them are given, including statistical staples such as logistic regression and naive Bayes, but also including more recent developments such as neural networks and recursive partitioning models. A key aspect is the development of methods for assessing performance of the models, and this is examined in detail. Keywords:},
author = {Hand, David J.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hand - 2001 - Modelling consumer credit risk.pdf:pdf},
journal = {IMA Journal of Management Mathematics},
keywords = {classification,credit scoring,creditworthiness},
pages = {139--155},
title = {{Modelling consumer credit risk}},
year = {2001}
}
@article{Snoek2014,
abstract = {Bayesian optimization has proven to be a highly effective methodology for the global optimization of unknown, expensive and multimodal functions. The ability to accurately model distributions over functions is critical to the effectiveness of Bayesian optimization. Although Gaussian processes provide a flexible prior over functions, there are various classes of functions that re- main difficult to model. One of the most frequently occurring of these is the class of non-stationary functions. The optimization of the hyperparameters of machine learning algorithms is a problem domain in which parameters are of- ten manually transformed a priori, for example by optimizing in “log-space,” to mitigate the effects of spatially-varying length scale.We develop a method- ology for automatically learning a wide family of bijective transformations or warpings of the input space using the Beta cumulative distribution function. We further extend the warping framework to multi-task Bayesian optimiza- tion so that multiple tasks can be warped into a jointly stationary space. On a set of challenging benchmark optimization tasks, we observe that the in- clusion of warping greatly improves on the state-of-the-art, producing better results faster and more reliably. 1.},
archivePrefix = {arXiv},
arxivId = {arXiv:1402.0929v3},
author = {Snoek, Jasper and Swersky, Kevin and Zemel, Richard and Adams, Ryan P},
eprint = {arXiv:1402.0929v3},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Snoek et al. - 2014 - Input Warping for Bayesian Optimization of Non-Stationary Functions.pdf:pdf},
isbn = {9781634393973},
journal = {Proceedings of the 31st International Conference on Machine Learning},
pages = {1674--1682},
title = {{Input Warping for Bayesian Optimization of Non-Stationary Functions}},
year = {2014}
}
@article{Tuv2009,
abstract = {Predictive models benefit from a compact, non-redundant subset of features that improves interpretability and generalization. Modern data sets are wide, dirty, mixed with both numerical and categorical predictors, and may contain interactive effects that require complex models. This is a challenge for filters, wrappers, and embedded feature selection methods. We describe details of an algorithm using tree-based ensembles to generate a compact subset of non-redundant features. Parallel and serial ensembles of trees are combined into a mixed method that can uncover masking and detect features of secondary effect. Simulated and actual examples illustrate the effectiveness of the approach.},
author = {Tuv, Eugene and Borisov, Alexander and Runger, George and Torkkola, Kori},
doi = {10.1145/1577069.1755828},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Tuv et al. - 2009 - Feature Selection with Ensembles , Artificial Variables , and Redundancy Elimination.pdf:pdf},
isbn = {1532-4435},
issn = {15324435},
journal = {Journal of Machine Learning Research},
keywords = {importance,masking,resampling,residuals,trees},
pages = {1341--1366},
title = {{Feature Selection with Ensembles , Artificial Variables , and Redundancy Elimination}},
url = {http://www.jmlr.org/papers/volume10/tuv09a/tuv09a.pdf},
volume = {10},
year = {2009}
}
@article{Shi2010,
author = {Shi, Zhan},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Shi - 2010 - Random Walks and Trees.pdf:pdf},
title = {{Random Walks and Trees}},
year = {2010}
}
@article{Friedman1999,
author = {Friedman, Jerome H},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Friedman - 1999 - Stochastic Gradient Boosting.pdf:pdf},
number = {3},
pages = {1--10},
title = {{Stochastic Gradient Boosting}},
volume = {1},
year = {1999}
}
@article{Criminisi2012,
abstract = {This review presents a unified, efficientmodel of random decision forests which can be applied to a number of machine learning, computer vision, and medical image analysis tasks. Our model extends existing forest-based techniques as it unifies classification, regression, density estimation, manifold learning, semi- supervised learning, and active learning under the same decision forest framework. This gives us the opportunity to write and optimize the core implementation only once, with application to many diverse tasks. The proposed model may be used both in a discriminative or gen- erative way and may be applied to discrete or continuous, labeled or unlabeled data. The main contributions of this review are: (1) Proposing a unified, probabilistic and efficient model for a variety of learning tasks; (2) Demonstrating margin-maximizing properties of classifica- tion forests; (3) Discussing probabilistic regression forests in compari- son with other nonlinear regression algorithms; (4) Introducing density forests for estimating probability density functions; (5) Proposing an efficient algorithm for sampling from a density forest; (6) Introducing manifold forests for nonlinear dimensionality reduction; (7) Proposing new algorithms for transductive learning and active learning. Finally, we discuss how alternatives such as random ferns and extremely ran- domized trees stem from our more general forest model. This document is directed at both students who wish to learn the basics of decision forests, as well as researchers interested in the new contributions. It presents both fundamental and novel concepts in a structured way, with many illustrative examples and real-world appli- cations. Thorough comparisons with state-of-the-art algorithms such as support vector machines, boosting and Gaussian processes are pre- sented and relative advantages and disadvantages discussed. The many synthetic examples and existing commercial applications demonstrate the validity of the proposed model and its flexibility},
author = {Criminisi, Antonio and Shotton, Jamie and Konukoglu, Ender},
doi = {10.1561/0600000035},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Criminisi, Shotton, Konukoglu - 2012 - Decision Forests A Unified Framework for Classification , Regression , Density Estimation , Mani.pdf:pdf},
number = {2011},
pages = {81--227},
title = {{Decision Forests : A Unified Framework for Classification , Regression , Density Estimation , Manifold Learning and Semi-Supervised Learning}},
volume = {7},
year = {2012}
}
@article{Ishwaran2007,
abstract = {We characterize and study variable importance (VIMP) and pairwise variable associations in binary regression trees. A key component involves the node mean squared error for a quantity we refer to as a maximal subtree. The theory naturally extends from single trees to ensembles of trees and applies to methods like random forests. This is useful because while importance values from random forests are used to screen variables, for example they are used to filter high throughput genomic data in Bioinformatics, very little theory exists about their properties.},
archivePrefix = {arXiv},
arxivId = {0711.2434},
author = {Ishwaran, Hemant},
doi = {10.1214/07-EJS039},
eprint = {0711.2434},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ishwaran - 2007 - Variable importance in binary regression trees and forests.pdf:pdf},
issn = {1935-7524},
keywords = {and phrases,cart,maximal subtree,random forests,received april 2007},
pages = {519--537},
title = {{Variable importance in binary regression trees and forests}},
url = {http://arxiv.org/abs/0711.2434},
volume = {1},
year = {2007}
}
@article{Snoek2013a,
abstract = {Advances in machine learning are having a profound impact on disciplines spanning the sciences. Assistive technology and health informatics are fields for which minor improvements achieved through leveraging more advanced machine learning algorithms can translate to major real world impact. However, successful application of machine learning currently requires broad domain knowledge to determine which model is appropriate for a given task, and model specific expertise to configure a model to a problem of interest. A major motivation for this thesis was: How can we make machine learning more accessible to assistive technology and health informatics researchers? Naturally, a complementary goal is to make machine learning more accessible in general. Specifically, in this thesis we explore how to automate the role of a machine learning expert through automatically adapting models and adjusting parameters to a given task of interest. This thesis consists of a number of contributions towards solving this challenging open problem in machine learning and these are empirically validated on four real-world applications. Through an interesting theoretical link between two seemingly disparate latent variable models, we create a hybrid model that allows one to flexibly interpolate over a parametric unsupervised neural network, a classification neural network and a non-parametric Gaussian process. We demonstrate empirically that this non-parametrically guided autoencoder allows one to learn a latent representation that is more useful for a given task of interest. We establish methods for automatically configuring machine learning model hyperpa- rameters using Bayesian optimization. We develop Bayesian methods for integrating over parameters, explore the use of different priors over functions, and develop methods to run experiments in parallel. We demonstrate empirically that these methods find better hyper- parameters on recent benchmark problems spanning machine learning in significantly less experiments than the methods employed by the problems’ authors. We further establish methods for incorporating parameter dependent variable cost in the optimization procedure. These methods find better hyperparameters in less cost, such as time, or within bounded cost, such as before a deadline. Additionally, we develop a constrained Bayesian optimization variant and demonstrate its superiority over the standard procedure in the presence of unknown constraints.},
author = {Snoek, Jasper},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Snoek - 2013 - Bayesian Optimization and Semiparametric Models with Applications to Assistive Technology.pdf:pdf},
title = {{Bayesian Optimization and Semiparametric Models with Applications to Assistive Technology}},
url = {https://tspace.library.utoronto.ca/handle/1807/43732},
year = {2013}
}
@article{Strobl2008,
abstract = {BACKGROUND: Random forests are becoming increasingly popular in many scientific fields because they can cope with "small n large p" problems, complex interactions and even highly correlated predictor variables. Their variable importance measures have recently been suggested as screening tools for, e.g., gene expression studies. However, these variable importance measures show a bias towards correlated predictor variables. RESULTS: We identify two mechanisms responsible for this finding: (i) A preference for the selection of correlated predictors in the tree building process and (ii) an additional advantage for correlated predictor variables induced by the unconditional permutation scheme that is employed in the computation of the variable importance measure. Based on these considerations we develop a new, conditional permutation scheme for the computation of the variable importance measure. CONCLUSION: The resulting conditional variable importance reflects the true impact of each predictor variable more reliably than the original marginal approach.},
author = {Strobl, Carolin and Boulesteix, Anne-Laure and Kneib, Thomas and Augustin, Thomas and Zeileis, Achim},
doi = {10.1186/1471-2105-9-307},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Strobl et al. - 2008 - Conditional variable importance for random forests.pdf:pdf},
isbn = {1471-2105},
issn = {1471-2105},
journal = {BMC bioinformatics},
pages = {307},
pmid = {18620558},
title = {{Conditional variable importance for random forests.}},
volume = {9},
year = {2008}
}
@article{Genuer2010a,
abstract = {This paper proposes, focusing on random forests, the increasingly used statistical method for classification and regression problems introduced by Leo Breiman in 2001, to investigate two classical issues of variable selection. The first one is to find important variables for interpretation and the second one is more restrictive and try to design a good parsimonious prediction model. The main contribution is twofold: to provide some experimental insights about the behavior of the variable importance index based on random forests and to propose a strategy involving a ranking of explanatory variables using the random forests score of importance and a stepwise ascending variable introduction strategy.},
author = {Genuer, Robin and Poggi, Jean-michel and Tuleau-Malot, Christine},
doi = {10.1016/j.patrec.2010.03.014},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Genuer, Poggi, Tuleau-Malot - 2010 - Variable selection using Random Forests.pdf:pdf;:Users/guillaume/Dropbox/Mendeley Desktop/Genuer, Poggi, Tuleau-Malot - 2010 - Variable selection using Random Forests(2).pdf:pdf},
isbn = {0167-8655},
issn = {01678655},
journal = {Pattern Recognition Letters},
keywords = {2010 msc,62g08,62g09,62h30,classification,high dimensional data,random forests,regression,variable importance,variable selection},
number = {14},
pages = {2225--2236},
title = {{Variable selection using Random Forests}},
volume = {31},
year = {2010}
}
@article{Lecun2006,
author = {Lecun, Yann and Chopra, Sumit and Hadsell, Raia and Ranzato, Marc Aurelio and Huang, Fu Jie},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Lecun et al. - 2006 - A Tutorial on Energy-Based Learning.pdf:pdf},
pages = {1--59},
title = {{A Tutorial on Energy-Based Learning}},
year = {2006}
}
@article{Huber1963,
author = {Huber, Peter J.},
doi = {10.1214/aoms/1177733256},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Huber - 1964 - Robust Estimation of a Location Parameter.pdf:pdf},
isbn = {0121202011},
issn = {0003-4851},
journal = {The Annals of Mathematical Statistics},
number = {1},
pages = {73--101},
title = {{Robust Estimation of a Location Parameter}},
volume = {35},
year = {1964}
}
@article{DelRio2014,
abstract = {In this age, big data applications are increasingly becoming the main focus of attention because of the enormous increment of data generation and storage that has taken place in the last years. This situation becomes a challenge when huge amounts of data are processed to extract knowledge because the data mining techniques are not adapted to the new space and time requirements. Furthermore, real-world data applications usually present a class distribution where the samples that belong to one class, which is precisely the main interest, are hugely outnumbered by the samples of the other classes. This circumstance, known as the class imbalance problem, complicates the learning process as the standard learning techniques do not correctly address this situation. In this work, we analyse the performance of several techniques used to deal with imbalanced datasets in the big data scenario using the Random Forest classifier. Specifically, oversampling, undersampling and cost-sensitive learning have been adapted to big data using MapReduce so that these techniques are able to manage datasets as large as needed providing the necessary support to correctly identify the underrepresented class. The Random Forest classifier provides a solid basis for the comparison because of its performance, robustness and versatility. An experimental study is carried out to evaluate the performance of the diverse algorithms considered. The results obtained show that there is not an approach to imbalanced big data classification that outperforms the others for all the data considered when using Random Forest. Moreover, even for the same type of problem, the best performing method is dependent on the number of mappers selected to run the experiments. In most of the cases, when the number of splits is increased, an improvement in the running times can be observed, however, this progress in times is obtained at the expense of a slight drop in the accuracy performance obtained. This decrement in the performance is related to the lack of density problem, which is evaluated in this work from the imbalanced data point of view, as this issue degrades the performance of classifiers in the imbalanced scenario more severely than in standard learning. © 2014 Elsevier Inc. All rights reserved.},
author = {del R\'{\i}o, Sara and L\'{o}pez, Victoria and Ben\'{\i}tez, Jos\'{e} Manuel and Herrera, Francisco},
doi = {10.1016/j.ins.2014.03.043},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/del R\'{\i}o et al. - 2014 - On the use of MapReduce for imbalanced big data using Random Forest.pdf:pdf},
isbn = {0020-0255},
issn = {00200255},
journal = {Information Sciences},
keywords = {Big data,Cost-sensitive learning,Imbalanced dataset,MapReduce,Random Forest,Sampling},
publisher = {Elsevier Inc.},
title = {{On the use of MapReduce for imbalanced big data using Random Forest}},
url = {http://dx.doi.org/10.1016/j.ins.2014.03.043},
year = {2014}
}
@phdthesis{Science2014,
abstract = {The subject of modelling rare events data is possibly very large, including supervised and unsupervised machine learning methods, non-linear curve fitting, non-parametric methods, etc. The scope of this thesis is limited to generalized linear models, notably binomial linear regression. This subject is addressed from a frequentist and a Bayesian perspective. The focus is on the use of different priors and link functions. The priors help to obtain reasonable and stable point estimates for the regression coefficients. Different link functions are investigated and their effect on obtaining good fit to imbalanced binary data. The report contains an elaboration of the theory and an application of various methods on real data using cross- validation.},
author = {{Van der Paal}, Bart},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Van der Paal - 2014 - A comparison of different methods for modelling rare events data.pdf:pdf},
title = {{A comparison of different methods for modelling rare events data}},
year = {2014}
}
@article{Ishwaran2008,
abstract = {We introduce random survival forests, a random forests method for the analysis of right-censored survival data. New survival splitting rules for growing survival trees are introduced, as is a new missing data algorithm for imputing missing data. A conservation-of-events principle for survival forests is introduced and used to define ensemble mortality, a simple interpretable measure of mortality that can be used as a predicted outcome. Several illustrative examples are given, including a case study of the prognostic implications of body mass for individuals with coronary artery disease. Computations for all examples were implemented using the freely available R-software package, randomSurvivalForest.},
archivePrefix = {arXiv},
arxivId = {0811.1645},
author = {Ishwaran, Hemant and Kogalur, Udaya B. and Blackstone, Eugene H. and Lauer, Michael S.},
doi = {10.1214/08-AOAS169},
eprint = {0811.1645},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ishwaran et al. - 2008 - Random survival forests.pdf:pdf},
issn = {19326157},
journal = {Annals of Applied Statistics},
keywords = {Conservation of events,Cumulative hazard function,Ensemble,Out-of-bag,Prediction error,Survival tree},
number = {3},
pages = {841--860},
pmid = {261057900003},
title = {{Random survival forests}},
volume = {2},
year = {2008}
}
@article{Hlosta2013,
author = {Hlosta, Martin and Str\'{\i}\v{z}, Rostislav and Kup\v{c}\'{\i}k, Jan and Zendulka, Jaroslav and Hru\v{s}ka, Tom\'{a}\v{s}},
doi = {10.7763/IJMLC.2013.V3.305},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hlosta et al. - 2013 - Constrained Classification of Large Imbalanced Data by Logistic Regression and Genetic Algorithm.pdf:pdf},
issn = {20103700},
journal = {International Journal of Machine Learning and Computing},
number = {2},
pages = {214--218},
title = {{Constrained Classification of Large Imbalanced Data by Logistic Regression and Genetic Algorithm}},
url = {http://www.ijmlc.org/index.php?m=content\&c=index\&a=show\&catid=36\&id=304},
volume = {3},
year = {2013}
}
@article{Hastie,
author = {Hastie, Trevor},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hastie - Unknown - Trees , Bagging , Random Forests and Boosting Two-class Classification.pdf:pdf},
title = {{Trees , Bagging , Random Forests and Boosting Two-class Classification}}
}
@article{Kyrillidis2014,
abstract = {We study the effectiveness of non-uniform randomized feature selection in decision tree classification. We experimentally evaluate two feature selection methodologies, based on information extracted from the provided dataset: (i) leverage scores-based and (ii) norm-based feature selection. Experimental evaluation of the proposed feature selection techniques indicate that such approaches might be more effective compared to naive uniform feature},
archivePrefix = {arXiv},
arxivId = {arXiv:1403.5877v1},
author = {Kyrillidis, Anastasios and Zouzias, Anastasios},
eprint = {arXiv:1403.5877v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kyrillidis, Zouzias - 2014 - Non-uniform Feature Sampling for Decision Tree Ensembles.pdf:pdf},
pages = {1--7},
title = {{Non-uniform Feature Sampling for Decision Tree Ensembles}},
year = {2014}
}
@article{Stekhoven2012,
abstract = {Modern data acquisition based on high-throughput technology is often facing the problem of missing data. Algorithms commonly used in the analysis of such large-scale data often depend on a complete set. Missing value imputation offers a solution to this problem. However, the majority of available imputation methods are restricted to one type of variable only: continuous or categorical. For mixed-type data the different types are usually handled separately. Therefore, these methods ignore possible relations between variable types. We propose a nonparametric method which can cope with different types of variables simultaneously. We compare several state of the art methods for the imputation of missing values. We propose and evaluate an iterative imputation method (missForest) based on a random forest. By averaging over many unpruned classification or regression trees random forest intrinsically constitutes a multiple imputation scheme. Using the built-in out-of-bag error estimates of random forest we are able to estimate the imputation error without the need of a test set. Evaluation is performed on multiple data sets coming from a diverse selection of biological fields with artificially introduced missing values ranging from 10\% to 30\%. We show that missForest can successfully handle missing values, particularly in data sets including different types of variables. In our comparative study missForest outperforms other methods of imputation especially in data settings where complex interactions and nonlinear relations are suspected. The out-of-bag imputation error estimates of missForest prove to be adequate in all settings. Additionally, missForest exhibits attractive computational efficiency and can cope with high-dimensional data.},
archivePrefix = {arXiv},
arxivId = {1105.0828},
author = {Stekhoven, Daniel J. and B\"{u}hlmann, Peter},
doi = {10.1093/bioinformatics/btr597},
eprint = {1105.0828},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Stekhoven, B\"{u}hlmann - 2012 - Missforest-Non-parametric missing value imputation for mixed-type data.pdf:pdf},
isbn = {1367-4811 (Electronic)$\backslash$n1367-4803 (Linking)},
issn = {13674803},
journal = {Bioinformatics},
number = {1},
pages = {112--118},
pmid = {22039212},
title = {{Missforest-Non-parametric missing value imputation for mixed-type data}},
volume = {28},
year = {2012}
}
@article{Ramdas,
abstract = {The idea of random projections is now pervasive in machine learning literature, useful in both improving our theoretical understanding of high-dimensional prob- lems and providing practical algorithms that have changed the field. In this survey, we hope to give the reader an introduction into this maginificent world of random projections, some of its curiosities and wonders, and its relations to other very important concepts in learning theory.},
author = {Ramdas, Aaditya},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ramdas - Unknown - A Random Introduction To Random Projections.pdf:pdf},
journal = {Citeseer},
pages = {1--5},
title = {{A Random Introduction To Random Projections}},
url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.377.2593\&rep=rep1\&type=pdf}
}
@misc{Scott2014,
author = {Scott, Clayton},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Scott - 2014 - Vapnik-Chevronenkis Theory.pdf:pdf},
pages = {1--11},
title = {{Vapnik-Chevronenkis Theory}},
year = {2014}
}
@article{Fantazzini2009,
abstract = {This paper extends the existing literature on empirical research in the field of credit risk default for Small Medium Enterprizes (SMEs). We propose a non- parametric approach based on Random Survival Forests (RSF) and we compare its performance with a standard logit model. To the authors’ knowledge, no studies in the area of credit risk default for SMEs have used a variety of statistical method- ologies to test the reliability of their predictions and to compare their performance against one another. As for the in-sample results, we find that our non-parametric model performs much better that the classical logit model. As for the out-of-sample performances, the evidence is just the opposite, and the logit performs better than the RSF model. We explain this evidence by showing how error in the estimates of default probabilities can affect classification error when the estimates are used in a classification rule. Keywords},
author = {Fantazzini, Dean and Figini, Silvia},
doi = {10.1007/s11009-008-9078-2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Fantazzini, Figini - 2009 - Random survival forests models for SME credit risk measurement.pdf:pdf},
issn = {13875841},
journal = {Methodology and Computing in Applied Probability},
keywords = {Classification,Credit risk,Default probability,Loss functions,Random survival trees},
number = {1},
pages = {29--45},
title = {{Random survival forests models for SME credit risk measurement}},
volume = {11},
year = {2009}
}
@article{Abdallah,
abstract = {Missing values in data are common in real world applications. Since the performance of many data mining algorithms depend critically on it being given a good metric over the input space, we decided in this paper to define a distance function for unlabeled datasets with missing values. We use the Bhattacharyya distance, which measures the similarity of two probability distributions, to define our new distance function. According to this distance, the distance between two points without missing attributes values is simply the Mahalanobis distance. When on the other hand there is a missing value of one of the coordinates, the distance is computed according to the distribution of the missing coordinate. Our distance is general and can be used as part of any algorithm that computes the distance between data points. Because its performance depends strongly on the chosen distance measure, we opted for the k nearest neighbor classifier to evaluate its ability to accurately reflect object similarity. We experimented on standard numerical datasets from the UCI repository from different fields. On these datasets we simulated missing values and compared the performance of the kNN classifier using our distance to other three basic methods. Our experiments show that kNN using our distance function outperforms the kNN using other methods. Moreover, the runtime performance of our method is only slightly higher than the other methods.},
author = {Abdallah, Loai and Shimshoni, Ilan},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Abdallah, Shimshoni - Unknown - A distance function for data with missing values and its application.pdf:pdf},
keywords = {bhattacharyya dis-,distance metric,missing values},
title = {{A distance function for data with missing values and its application}},
url = {http://www.metro450.org.il/Metro450/Templates/showpage.asp?DBID=1\&LNGID=1\&TMID=178\&FID=1132\&PID=0\&IID=1437}
}
@article{Caruana2004,
author = {Caruana, Rich and Niculescu-Mizil, Alexandru},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Caruana, Niculescu-Mizil - 2004 - An empirical evaluation of supervised learning for ROC area.pdf:pdf},
journal = {The First Workshop on ROC Analysis in AI},
title = {{An empirical evaluation of supervised learning for ROC area}},
url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.60.1947\&rep=rep1\&type=pdf},
year = {2004}
}
@incollection{Johnson1984a,
author = {Johnson, William B. and Lindenstrauss, Joram},
doi = {10.1090/conm/026/737400},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Johnson, Lindenstrauss - 1984 - Extensions of Lipschitz mappings into a Hilbert space.pdf:pdf},
pages = {189--206},
title = {{Extensions of Lipschitz mappings into a Hilbert space}},
url = {http://www.ams.org/conm/026/},
year = {1984}
}
@article{Ba2015,
abstract = {We present an attention-based model for recognizing multiple objects in images. The proposed model is a deep recurrent neural network trained with reinforcement learning to attend to the most relevant regions of the input image. We show that the model learns to both localize and recognize multiple objects despite being given only class labels during training. We evaluate the model on the challenging task of transcribing house number sequences from Google Street View images and show that it is both more accurate than the state-of-the-art convolutional networks and uses fewer parameters and less computation.},
archivePrefix = {arXiv},
arxivId = {arXiv:1412.7755v2},
author = {Ba, Jimmy Lei and Mnih, Volodymyr and Kavukcuoglu, Koray},
eprint = {arXiv:1412.7755v2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ba, Mnih, Kavukcuoglu - 2015 - Multiple Object Recognition With Visual Attention.pdf:pdf},
journal = {ICLR},
pages = {1--10},
title = {{Multiple Object Recognition With Visual Attention}},
year = {2015}
}
@article{Bischl2012a,
abstract = {Meta-modeling has become a crucial tool in solving expensive optimization problems. Much of the work in the past has focused on finding a good regression method to model the fitness function. Examples include classical linear regression, splines, neural networks, Kriging and support vector regression. This paper specifically draws attention to the fact that assessing model accuracy is a crucial aspect in the meta-modeling framework. Resampling strategies such as cross-validation, subsampling, bootstrapping, and nested resampling are prominent methods for model validation and are systematically discussed with respect to possible pitfalls, shortcomings, and specific features. A survey of meta-modeling techniques within evolutionary optimization is provided. In addition, practical examples illustrating some of the pitfalls associated with model selection and performance assessment are presented. Finally, recommendations are given for choosing a model validation technique for a particular setting.},
author = {Bischl, B. and Mersmann, O. and Trautmann, H. and Weihs, C.},
doi = {10.1162/EVCO\_a\_00069},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bischl et al. - 2012 - Resampling Methods for Meta-Model Validation with Recommendations for Evolutionary Computation.pdf:pdf},
isbn = {1063-6560},
issn = {1063-6560},
journal = {Evolutionary Computation},
keywords = {evolutionary computation,evolutionary optimization,meta-models,model validation,regression,resampling},
number = {2},
pages = {249--275},
pmid = {22339368},
title = {{Resampling Methods for Meta-Model Validation with Recommendations for Evolutionary Computation}},
volume = {20},
year = {2012}
}
@book{Koller2009,
author = {Koller, Daphne and Friedman, Nir},
booktitle = {Cs-Wwwarchiv.Cs.Unibas.Ch},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Koller, Friedman - 2009 - Probabilistic Graphical Models Principles and Techniques.pdf:pdf},
isbn = {9780262013192},
title = {{Probabilistic Graphical Models: Principles and Techniques}},
year = {2009}
}
@article{Duvenaud2015,
archivePrefix = {arXiv},
arxivId = {arXiv:1502.03492v3},
author = {Duvenaud, David and Adams, Ryan P},
eprint = {arXiv:1502.03492v3},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Duvenaud, Adams - 2015 - Gradient-based Hyperparameter Optimization through Reversible Learning.pdf:pdf},
title = {{Gradient-based Hyperparameter Optimization through Reversible Learning}},
year = {2015}
}
@article{Zhao2011,
author = {Zhao, Peilin and Hoi, Steven C H and Jin, Rong and Yang, Tianbao},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Zhao et al. - 2011 - Online AUC Maximization.pdf:pdf},
isbn = {978-1-4503-0619-5},
journal = {Proceedings of the 28th International Conference on Machine Learning (ICML-11)},
pages = {233--240},
title = {{Online AUC Maximization}},
year = {2011}
}
@article{Sega2011,
abstract = {Random forests have emerged as a versatile and highly accurate classification and regression methodology, requiring little tuning and providing interpretable outputs. Here, we briefly outline the genesis of, and motivation for, the random forest paradigm as an outgrowth from earlier tree-structured techniques. We elaborate on aspects of prediction error and attendant tuning parameter issues. However, our emphasis is on extending the random forest schema to the multiple response setting. We provide a simple illustrative example from ecology that showcases the improved fit and enhanced interpretation afforded by the random forest framework. © 2011 John Wiley \& Sons, Inc. WIREs Data Mining Knowl Discov 2011 1 80-87 DOI: 10.1002/widm.12},
author = {Segal, Mark and Xiao, Yuanyuan},
doi = {10.1002/widm.12},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Segal, Xiao - 2011 - Multivariate random forests.pdf:pdf},
isbn = {1942-4795},
issn = {19424787},
journal = {Wiley Interdisciplinary Reviews: Data Mining and Knowledge Discovery},
number = {1},
pages = {80--87},
title = {{Multivariate random forests}},
volume = {1},
year = {2011}
}
@article{Hido2009b,
abstract = {The class imbalance problem appears in many real-world applications of classification learning. We propose an ensemble algorithm “Roughly Balanced (RB) Bagging” using a novel sampling technique to improve the original bagging algorithm for data sets with skewed class distributions. For this sampling method, the number of samples in the largest and smallest classes are different, but they are effectively balanced when averaged over all of the subsets, which supports the approach of bagging in a more appropriate way. Individual models in RB Bagging tend to show larger diversity, which is one of the keys of ensemble models, compared with existing bagging-based methods for imbalanced data that use exactly the same number of majority and minority examples for every training subset. In addition, the proposed method makes full use of all of the minority examples by under-sampling, which is efficiently done by using negative binomial distributions. Numerical experiments using benchmark and real-world data sets demonstrate that RB Bagging shows better performance than the existing “balanced” methods and other common methods for area under the ROC curve (AUC), which is a widely used metric in the class imbalance problem.},
author = {Hido, Shohei and Kashima, Hisashi and Takahashi, Yutaka},
doi = {10.1002/sam.10061},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hido, Kashima, Takahashi - 2009 - Roughly balanced Bagging for Imbalanced data.pdf:pdf},
isbn = {9781605603179},
issn = {19321872},
journal = {Statistical Analysis and Data Mining},
keywords = {Bagging,Imbalanced data,Negative binomial distribution,Resampling},
number = {5-6},
pages = {412--426},
title = {{Roughly balanced Bagging for Imbalanced data}},
volume = {2},
year = {2009}
}
@article{Nikulin2009,
abstract = {With imbalanced data a classifier built using all of the data has the tendency to ignore the minority class. To overcome this problem, we propose to use an ensemble classifier constructed on the basis of a large number of relatively small and balanced subsets, where representatives from both patterns are to be selected randomly. As an outcome, the system produces the matrix of linear regression coefficients whose rows represent the random sub- sets and the columns represent the features. Based on this matrix, we make an assessment of how stable the influence of a particular feature is. It is proposed to keep in themodel only features with stable influence. The final model represents an average of the base-learners, which is not necessarily a linear regression. Proper data pre-processing is very important for the effectiveness of the whole system, and it is proposed to reduce the original data to the most simple binary sparse format, which is particularly convenient for the construction of decision trees. As a result, any particular feature will be represented by several binary variables or bins, which are absolutely equivalent in terms of data structure. This property is very important and may be used for feature selection. The proposed method exploits not only contributions of particular variables to the base-learners, but also the diversity of such contributions. Test results against KDD-2009 competition datasets are presented. Keywords:},
author = {Nikulin, Vladimir and McLachlan, Geoffrey J.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Nikulin, McLachlan - 2009 - Classification of Imbalanced Marketing Data with Balanced Random Sets.pdf:pdf},
journal = {Journal of Machine Learning Research- \ldots},
keywords = {boosting,decision trees,ensemble classifier,gradient-based optimisation,matrix factorisation,random forests},
pages = {89--100},
title = {{Classification of Imbalanced Marketing Data with Balanced Random Sets.}},
url = {http://jmlr.org/proceedings/papers/v7/nikulin09/nikulin09.pdf},
year = {2009}
}
@article{Kubat1997,
abstract = {Adding examples of the majority class to the training set can have a detrimental effect on the learner's behavior: noisy or otherwise unreliable examples from the majority class can overwhelm the minority class. The paper discusses criteria to evaluate the utility of classifiers induced from such imbalanced training sets, gives explanation of the poor behavior of some learners under these circumstances, and suggests as a solution a simple technique called one-sided selection of...},
author = {Kubat, M. and Matwin, S.},
doi = {10.1.1.43.4487},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kubat, Matwin - 1997 - Addressing the curse of imbalanced training sets one-sided selection.pdf:pdf},
isbn = {1-55860-486-3},
journal = {Icml},
pages = {179--186},
pmid = {297143},
title = {{Addressing the curse of imbalanced training sets: one-sided selection}},
volume = {97},
year = {1997}
}
@article{Witzany2010,
abstract = {The paper proposes an application of the survival time analysis methodology to estimations of the Loss Given Default (LGD) parameter. The main advantage of the survival analysis approach compared to classical regression methods is that it allows exploiting partial recovery data. The model is also modified in order to improve performance of the appropriate goodness of fit measures. The empirical testing shows that the Cox proportional model applied to LGD modeling performs better than the linear and logistic regressions. In addition a significant improvement is achieved with the modified pseudo Cox LGD model.},
author = {Witzany, Jiri and Rychnovsky, Michal and Charamza, Pavel},
doi = {10.1080/15732471003594393},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Witzany, Rychnovsky, Charamza - 2010 - Survival Analysis in LGD Modeling.pdf:pdf},
journal = {Structure and Infrastructure Engineering},
keywords = {correlation,credit risk,loss given default,recovery rate,regulatory capital},
pages = {1--18},
title = {{Survival Analysis in LGD Modeling}},
year = {2010}
}
@article{Lu2010,
author = {Lu, X and Tang, K and Yao, X},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Lu, Tang, Yao - 2010 - Evolving Neural Networks with Maximum AUC for Imbalanced Data Classification.pdf:pdf},
journal = {Hybrid Artificial Intelligence Systems},
keywords = {auc,class-imbalance learning,differential evolution,evolutionary algorithms,feed-forward neural,networks,roc},
pages = {335--342},
title = {{Evolving Neural Networks with Maximum AUC for Imbalanced Data Classification}},
url = {http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=pubmed\&cmd=Retrieve\&dopt=AbstractPlus\&list\_uids=related:XJpSBnxlorwJ},
year = {2010}
}
@article{Hart1968,
abstract = {Not Available},
author = {Hart, P.},
doi = {10.1109/TIT.1968.1054155},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hart - 1968 - The condensed nearest neighbor rule (Corresp.).pdf:pdf},
isbn = {1595931805},
issn = {0018-9448},
journal = {IEEE Transactions on Information Theory},
number = {3},
pages = {1966--1967},
title = {{The condensed nearest neighbor rule (Corresp.)}},
volume = {14},
year = {1968}
}
@article{Menon2007,
abstract = {Bachelor of Science},
author = {Menon, Aditya Krishna},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Menon - 2007 - Random projections and applications to dimensionality reduction.pdf:pdf},
number = {March 2007},
title = {{Random projections and applications to dimensionality reduction}},
url = {http://cseweb.ucsd.edu/~akmenon/HonoursThesis.pdf},
year = {2007}
}
@article{Liu2013,
abstract = {Node splitting is an important issue in Random Forest but robust $\backslash$nsplitting requires a large number of training samples. Existing solutions fail $\backslash$nto properly partition the feature space if there are insufficient training data. $\backslash$nIn this paper, we present semi-supervised splitting to overcome this limitation $\backslash$nby splitting nodes with the guidance of both labeled and unlabeled data. In $\backslash$nparticular, we derive a nonparametric algorithm to obtain an accurate quality $\backslash$nmeasure of splitting by incorporating abundant unlabeled data. To avoid the $\backslash$ncurse of dimensionality, we project the data points from the original $\backslash$nhigh-dimensional feature space onto a low-dimensional subspace before $\backslash$nestimation. A unified optimization framework is proposed to select a coupled $\backslash$npair of subspace and separating hyper plane such that the smoothness of the $\backslash$nsubspace and the quality of the splitting are guaranteed simultaneously. The $\backslash$nproposed algorithm is compared with state-of-the-art supervised and $\backslash$nsemi-supervised algorithms for typical computer vision applications such as $\backslash$nobject categorization and image segmentation. Experimental results on publicly $\backslash$navailable datasets demonstrate the superiority of our method.},
author = {Liu, Xiao and Song, Mingli and Tao, Dacheng and Liu, Zicheng and Zhang, Luming and Chen, Chun and Bu, Jiajun},
doi = {10.1109/CVPR.2013.70},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Liu et al. - 2013 - Semi-supervised node splitting for random forest construction.pdf:pdf},
isbn = {978-0-7695-4989-7},
issn = {10636919},
journal = {Proceedings of the IEEE Computer Society Conference on Computer  Vision and Pattern Recognition},
keywords = {node splitting,random forest,semi-supervised learning},
pages = {492--499},
title = {{Semi-supervised node splitting for random forest construction}},
year = {2013}
}
@article{Chawla2002a,
abstract = {An approach to the construction of classifiers from imbalanced datasets is described. A dataset is imbalanced if the classification categories are not approximately equally rep- resented. Often real-world data sets are predominately composed of “normal” examples with only a small percentage of “abnormal” or “interesting” examples. It is also the case that the cost of misclassifying an abnormal (interesting)example as a normal example is often much higher than the cost of the reverse error. Under-sampling of the majority (nor- mal)class has been proposed as a good means of increasing the sensitivity of a classifier to the minority class. This paper shows that a combination of our method of over-sampling the minority (abnormal)class and under-sampling the majority (normal) class can achieve better classifier performance (in ROC space)than only under-sampling the majority class. This paper also shows that a combination of ourmethod of over-sampling the minority class and under-sampling the majority class can achieve better classifier performance (in ROC space)than varying the loss ratios in Ripper or class priors in Naive Bayes. Our method of over-sampling the minority class involves creating synthetic minority class examples. Experiments are performed using C4.5, Ripper and a Naive Bayes classifier. The method is evaluated using the area under the Receiver Operating Characteristic curve (AUC)and the ROC convex hull strategy. 1.},
author = {Chawla, Nitesh V. and Bowyer, Kevin W. and Hall, Lawrence O. and Kegelmeyer, W. Philip},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Chawla et al. - 2002 - SMOTE Synthetic Minority Over-sampling Technique.pdf:pdf},
pages = {321--357},
title = {{SMOTE : Synthetic Minority Over-sampling Technique}},
volume = {16},
year = {2002}
}
@article{DalPozzolo2013,
author = {{Dal Pozzolo}, Andrea and Caelen, Olivier and Waterschoot, Serge and Bontempi, Gianluca},
doi = {10.1007/978-3-642-41278-3\_4},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Dal Pozzolo et al. - 2013 - Racing for unbalanced methods selection.pdf:pdf},
isbn = {9783642412776},
issn = {03029743},
journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
keywords = {Fraud detection,Racing,Unbalanced data},
pages = {24--31},
title = {{Racing for unbalanced methods selection}},
volume = {8206 LNCS},
year = {2013}
}
@article{Kapelner2013,
archivePrefix = {arXiv},
arxivId = {arXiv:1312.2171v3},
author = {Kapelner, Adam and Bleich, Justin},
eprint = {arXiv:1312.2171v3},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kapelner, Bleich - 2013 - bartMachine Machine Learning With Bayesian Additive Regression Trees.pdf:pdf},
journal = {ArXiv e-prints},
keywords = {bayesian,java,machine learning,non-parametric,r,statistical learning},
number = {2013},
title = {{bartMachine: Machine Learning With Bayesian Additive Regression Trees}},
year = {2013}
}
@article{Menardi2010,
abstract = {The problem of modeling binary responses by using cross-sectional data has been addressed with a number of satisfying solutions that draw on both parametric and nonparametric methods. However, there exist many real situations where one of the two responses (usually the most interesting for the analysis) is rare. It has been largely reported that this class imbalance heavily compromises the process of learning, because the model tends to focus on the prevalent class and to ignore the rare events. However, not only the estimation of the classification model is affected by a skewed distribution of the classes, but also the evaluation of its accuracy is jeopardized, because the scarcity of data leads to poor estimates of the model’s accuracy. In this work, the effects of class imbalance on model training and model assessing are discussed. Moreover, a unified and systematic framework for dealing with the problem of imbalanced classification is proposed, based on a smoothed bootstrap re-sampling technique. The proposed technique is founded on a sound theoretical basis and an extensive empirical study shows that it outperforms the main other remedies to face imbalanced learning problems.},
author = {Menardi, Giovanna and Torelli, Nicola},
doi = {10.1007/s10618-012-0295-5},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Menardi, Torelli - 2014 - Training and assessing classification rules with imbalanced data(2).pdf:pdf;:Users/guillaume/Dropbox/Mendeley Desktop/Menardi, Torelli - 2014 - Training and assessing classification rules with imbalanced data.pdf:pdf},
isbn = {9788883033216},
issn = {13845810},
journal = {Data Mining and Knowledge Discovery},
keywords = {Accuracy,Binary classification,Bootstrap,Imbalanced learning,Kernel density estimation},
number = {1},
pages = {1--28},
title = {{Training and assessing classification rules with imbalanced data}},
url = {http://etabeta.univ.trieste.it/dspace/handle/10077/4002},
volume = {28},
year = {2014}
}
@article{Ioffe2015,
archivePrefix = {arXiv},
arxivId = {arXiv:1502.03167v3},
author = {Ioffe, Sergey and Szegedy, Christian},
eprint = {arXiv:1502.03167v3},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ioffe, Szegedy - 2015 - Batch Normalization Accelerating Deep Network Training by Reducing Internal Covariate Shift.pdf:pdf},
journal = {arXiv preprint arXiv:1502.03167v3},
title = {{Batch Normalization : Accelerating Deep Network Training by Reducing Internal Covariate Shift}},
year = {2015}
}
@article{Han2005,
abstract = {In recent years, mining with imbalanced data sets receives more and more attentions in both theoretical and practical aspects. This paper introduces the importance of imbalanced data sets and their broad application domains in data mining, and then summarizes the evaluation metrics and the existing methods to evaluate and solve the imbalance problem. Synthetic minority over-sampling technique (SMOTE) is one of the over-sampling methods addressing this problem. Based on SMOTE method, this paper presents two new minority over-sampling methods, borderline-SMOTE1 and borderline-SMOTE2, in which only the minority examples near the borderline are over-sampled. For the minority class, experiments show that our approaches achieve better TP rate and F-value than SMOTE and random over-sampling methods.},
author = {Han, Hui and Wang, Wy Wen-Yuan Wy and Mao, Bh Bing-huan},
doi = {10.1007/11538059\_91},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Han, Wang, Mao - 2005 - Borderline-SMOTE A new over-sampling method in imbalanced data sets learning.pdf:pdf},
isbn = {978-3-540-28226-6},
issn = {1941-0506},
journal = {Advances in intelligent computing},
keywords = {High dimensional data,Multidimensional projection,Visual data mining,classification,imbalance,oversampling,smote},
number = {12},
pages = {878--887},
pmid = {22034378},
title = {{Borderline-SMOTE: A new over-sampling method in imbalanced data sets learning}},
url = {http://dx.doi.org/10.1007/11538059\_91$\backslash$nhttp://link.springer.com/chapter/10.1007/11538059\_91$\backslash$nhttp://link.springer.com/chapter/10.1007/11538059\_91$\backslash$nhttp://www.bmva.org/bmvc/1988/avc-88-023.html},
volume = {17},
year = {2005}
}
@article{Fawcett2006,
abstract = {Receiver operating characteristics (ROC) graphs are useful for organizing classifiers and visualizing their performance. ROC graphs are commonly used in medical decision making, and in recent years have been used increasingly in machine learning and data mining research. Although ROC graphs are apparently simple, there are some common misconceptions and pitfalls when using them in practice. The purpose of this article is to serve as an introduction to ROC graphs and as a guide for using them in research. ?? 2005 Elsevier B.V. All rights reserved.},
author = {Fawcett, Tom},
doi = {10.1016/j.patrec.2005.10.010},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Fawcett - 2006 - An introduction to ROC analysis.pdf:pdf},
isbn = {226},
issn = {01678655},
journal = {Pattern Recognition Letters},
keywords = {Classifier evaluation,Evaluation metrics,ROC analysis},
number = {8},
pages = {861--874},
title = {{An introduction to ROC analysis}},
volume = {27},
year = {2006}
}
@article{Kearns1998,
abstract = {In this work, we present a new bottom-up algorithm for decision tree pruning that is very efficient (requiring only a single pass through the given tree), and prove a strong performance guarantee for the generalization error of the resulting pruned tree. We work in the typical setting in which the given tree T may have been derived from the given training sample S, and thus may badly overfit S. In this setting, we give bounds on the amount of additional generalization error that our pruning suffers compared to the optimal pruning of T. More generally, our results show that if there is a pruning of T with small error, and whose size is small compared to jSj, then our algorithm will find a pruning whose error is not much larger. This style of result has been called an index of resolvability result by Barron and Cover in the context of density estimation. A novel feature of our algorithm is its locality-- the decision to prune a subtree is based entirely on properties of that subtree and the sample reaching it. To analyze our algorithm, we develop tools of local uniform convergence, a generalization of the standard notion that may prove useful in other settings. Citations},
author = {Kearns, Mj and Mansour, Yishay},
doi = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.32.6464\&rep=rep1\&type=pdf},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kearns, Mansour - 1998 - A Fast, Bottom-Up Decision Tree Pruning Algorithm with Near-Optimal Generalization.pdf:pdf},
journal = {ICML},
keywords = {decision trees,model selection,pruning,theoretical analysis,uni},
pages = {269--277},
title = {{A Fast, Bottom-Up Decision Tree Pruning Algorithm with Near-Optimal Generalization.}},
url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.32.6464\&rep=rep1\&type=pdf},
year = {1998}
}
@article{Meinshausen2006,
abstract = {Random forests were introduced as a machine learning tool in Breiman (2001) and have since proven to be very popular and powerful for high-dimensional regression and classifi- cation. For regression, random forests give an accurate approximation of the conditional mean of a response variable. It is shown here that random forests provide information about the full conditional distribution of the response variable, not only about the con- ditional mean. Conditional quantiles can be inferred with quantile regression forests, a generalisation of random forests. Quantile regression forests give a non-parametric and accurate way of estimating conditional quantiles for high-dimensional predictor variables. The algorithm is shown to be consistent. Numerical examples suggest that the algorithm is competitive in terms of predictive power.},
author = {Meinshausen, Nicolai},
doi = {10.1111/j.1541-0420.2010.01521.x},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Meinshausen - 2006 - Quantile Regression Forests.pdf:pdf},
isbn = {1532-4435},
issn = {15410420},
journal = {Journal of Machine Learning Research},
keywords = {adaptive neighborhood regression,quantile regression,random forests},
pages = {983--999},
pmid = {21133883},
title = {{Quantile Regression Forests}},
volume = {7},
year = {2006}
}
@article{Natekin2013,
abstract = {Gradient boosting machines are a family of powerful machine-learning techniques that have shown considerable success in a wide range of practical applications. They are highly customizable to the particular needs of the application, like being learned with respect to different loss functions. This article gives a tutorial introduction into the methodology of gradient boosting methods with a strong focus on machine learning aspects of modeling. A theoretical information is complemented with descriptive examples and illustrations which cover all the stages of the gradient boosting model design. Considerations on handling the model complexity are discussed. Three practical examples of gradient boosting applications are presented and comprehensively analyzed.},
author = {Natekin, Alexey and Knoll, Alois},
doi = {10.3389/fnbot.2013.00021},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Natekin, Knoll - 2013 - Gradient boosting machines, a tutorial.pdf:pdf},
issn = {16625218},
journal = {Frontiers in Neurorobotics},
keywords = {Boosting,Classification,Gradient boosting,Machine learning,Regression,Robotic control,Text classification},
number = {DEC},
pmid = {24409142},
title = {{Gradient boosting machines, a tutorial}},
volume = {7},
year = {2013}
}
@article{Degorski2008,
abstract = {The article discusses methods of improving the ways of applying balanced random forests (BRFs), a machine learning classification algorithm, used to extract definitions from written texts. These methods include different approaches to selecting attributes, optimising the classifier prediction threshold for the task of definition extraction and initial filtering by a very simple grammar.},
author = {Deg\'{o}rski, Lukasz and Kobyliński, Łukasz and Przepi\'{o}rkowski, Adam},
doi = {10.1109/IMCSIT.2008.4747264},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Deg\'{o}rski, Kobyliński, Przepi\'{o}rkowski - 2008 - Definition extraction Improving balanced random forests.pdf:pdf},
isbn = {9788360810149},
issn = {1896-7094},
journal = {Proceedings of the International Multiconference on Computer Science and Information Technology, IMCSIT 2008},
pages = {353--357},
pmid = {16615219},
title = {{Definition extraction: Improving balanced random forests}},
volume = {3},
year = {2008}
}
@article{LeCun2015,
author = {LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
doi = {10.1038/nature14539},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/LeCun, Bengio, Hinton - 2015 - Deep learning.pdf:pdf},
issn = {0028-0836},
journal = {Nature},
number = {7553},
pages = {436--444},
title = {{Deep learning}},
url = {http://www.nature.com/doifinder/10.1038/nature14539},
volume = {521},
year = {2015}
}
@article{Manuscript2012,
author = {Manuscript, Author and Machines, Nonparametric Learning},
doi = {10.3414/ME00-01-0052.Probability},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Manuscript, Machines - 2012 - NIH Public Access.pdf:pdf},
keywords = {brier score,consistency,k nearest neighbor,logistic regression,probability,random forest},
number = {1},
pages = {74--81},
title = {{NIH Public Access}},
volume = {51},
year = {2012}
}
@article{Japkowicz2000,
abstract = {With the continuous expansion of data availability in many large-scale, complex, and networked systems, such as surveillance, security, Internet, and finance, it becomes critical to advance the fundamental understanding of knowledge discovery and analysis from raw data to support decision-making processes. Although existing knowledge discovery and data engineering techniques have shown great success in many real-world applications, the problem of learning from imbalanced data (the imbalanced learning problem) is a relatively new challenge that has attracted growing attention from both academia and industry. The imbalanced learning problem is concerned with the performance of learning algorithms in the presence of underrepresented data and severe class distribution skews. Due to the inherent complex characteristics of imbalanced data sets, learning from such data requires new understandings, principles, algorithms, and tools to transform vast amounts of raw data efficiently into information and knowledge representation. In this paper, we provide a comprehensive review of the development of research in learning from imbalanced data. Our focus is to provide a critical review of the nature of the problem, the state-of-the-art technologies, and the current assessment metrics used to evaluate learning performance under the imbalanced learning scenario. Furthermore, in order to stimulate future research in this field, we also highlight the major opportunities and challenges, as well as potential important research directions for learning from imbalanced data.},
author = {He, Haibo and Garcia, Edwardo A.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/He, Garcia - 2009 - Learning from Imbalanced Data.pdf:pdf},
journal = {IEEE TRANSACTIONS ON KNOWLEDGE AND DATA ENGINEERING},
number = {9},
title = {{Learning from Imbalanced Data}},
url = {http://www.aaai.org/Papers/Workshops/2000/WS-00-05/WS00-05-003.pdf},
volume = {21},
year = {2009}
}
@phdthesis{Ciss2013,
abstract = {Dans le cadre des m\'{e}thodes d’apprentissage ensemblistes, nous pr\'{e}sentons les for\^{e}ts uniform\'{e}ment al\'{e}atoires, une variante de l’algorithme de r\'{e}f\'{e}rence "Random Forests" (Breiman (2001)). Dans le cadre de la classification, nous montrons sa convergence vers l’erreur de Bayes, en nous aidant des travaux de Devroye, Gy\"{o}rfi et Lugosi (1996), puis de ceux de Biau, Devroye et Lugosi (2008). Les for\^{e}ts unifor- m\'{e}ment al\'{e}atoires h\'{e}ritent des m\^{e}mes propri\'{e}t\'{e}s th\'{e}oriques que les for\^{e}ts al\'{e}atoires de Breiman. Dans la pratique, le caract\`{e}re incr\'{e}mental, l’extrapolation ou la s\'{e}lec- tion locale de variables font partie des nouveaux outils int\'{e}gr\'{e}s dans le mod\`{e}le et \'{e}tendent ses applications. L’algorithme est disponible sous la forme d’un package R.},
author = {Ciss, Sa\"{\i}p},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ciss - 2013 - For\^{e}ts uniform\'{e}ment al\'{e}atoires.pdf:pdf},
number = {1998},
pages = {1--63},
title = {{For\^{e}ts uniform\'{e}ment al\'{e}atoires}},
year = {2013}
}
@article{Ferri2002,
author = {Ferri, C\`{e}sar and Flach, Peter and Hern\'{a}ndez-Orallo, J},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ferri, Flach, Hern\'{a}ndez-Orallo - 2002 - Learning decision trees using the area under the ROC curve.pdf:pdf},
journal = {International Conference on Machine Learning},
pages = {139--146},
title = {{Learning decision trees using the area under the ROC curve}},
url = {http://www2.cs.ust.hk/~qyang/537/Papers/flachICML02.pdf},
year = {2002}
}
@article{Biau,
abstract = {This article addresses the problem of supervised classification of Cox pro- cess trajectories, whose random intensity is driven by some exogenous random covariable. The classification task is achieved through a regularized convex empirical risk minimiza- tion procedure, and a nonasymptotic oracle inequality is derived. We show that the algorithm provides a Bayes-risk consistent classifier. Furthermore, it is proved that the classifier converges at a rate which adapts to the unknown regularity of the intensity pro- cess. Our results are obtained by taking advantage of martingale and stochastic calculus arguments, which are natural in this context and fully exploit the functional nature of the problem.},
author = {Biau, G\'{e}rard and Cadre, Beno\^{\i}t and Paris, Quentin},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Biau, Cadre, Paris - Unknown - Classification Supervis\'{e}e de Processus de Cox.pdf:pdf},
keywords = {consistency,cox process,oracle inequality,regularization,stochastic calculus,supervised classification},
pages = {1--6},
title = {{Classification Supervis\'{e}e de Processus de Cox}}
}
@article{Tripoliti2010,
abstract = {The aim of this work is to present an automated method that assists in the diagnosis of Alzheimer's disease and also supports the monitoring of the progression of the disease. The method is based on features extracted from the data acquired during an fMRI experiment. It consists of six stages: (a) preprocessing of fMRI data, (b) modeling of fMRI voxel time series using a Generalized Linear Model, (c) feature extraction from the fMRI data, (d) feature selection, (e) classification using classical and improved variations of the Random Forests algorithm and Support Vector Machines, and (f) conversion of the trees, of the Random Forest, to rules which have physical meaning. The method is evaluated using a dataset of 41 subjects. The results of the proposed method indicate the validity of the method in the diagnosis (accuracy 94\%) and monitoring of the Alzheimer's disease (accuracy 97\% and 99\%). © 2009 Elsevier Inc. All rights reserved.},
author = {Tripoliti, Evanthia E. and Fotiadis, Dimitrios I. and Argyropoulou, Maria and Manis, George},
doi = {10.1016/j.jbi.2009.10.004},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Tripoliti et al. - 2010 - A six stage approach for the diagnosis of the Alzheimer's disease based on fMRI data.pdf:pdf},
issn = {15320464},
journal = {Journal of Biomedical Informatics},
keywords = {Alzheimer's disease,Classification,Functional magnetic resonance imaging,Random Forests,Support Vector Machines},
number = {2},
pages = {307--320},
pmid = {19883796},
publisher = {Elsevier Inc.},
title = {{A six stage approach for the diagnosis of the Alzheimer's disease based on fMRI data}},
url = {http://dx.doi.org/10.1016/j.jbi.2009.10.004},
volume = {43},
year = {2010}
}
@article{Nanni2009,
abstract = {In this paper, we investigate the performance of several systems based on ensemble of classifiers for bankruptcy prediction and credit scoring. The obtained results are very encouraging, our results improved the performance obtained using the stand-alone classifiers. We show that the method "Random Subspace" outperforms the other ensemble methods tested in this paper. Moreover, the best stand-alone method is the multi-layer perceptron neural net, while the best method tested in this work is the Random Subspace of Levenberg-Marquardt neural net. In this work, three financial datasets are chosen for the experiments: Australian credit, German credit, and Japanese credit. ?? 2008 Elsevier Ltd. All rights reserved.},
author = {Nanni, Loris and Lumini, Alessandra},
doi = {10.1016/j.eswa.2008.01.018},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Nanni, Lumini - 2009 - An experimental comparison of ensemble of classifiers for bankruptcy prediction and credit scoring.pdf:pdf},
isbn = {0957-4174},
issn = {09574174},
journal = {Expert Systems with Applications},
keywords = {Bankruptcy prediction,Credit scoring,Ensemble of classifiers},
number = {2 PART 2},
pages = {3028--3033},
publisher = {Elsevier Ltd},
title = {{An experimental comparison of ensemble of classifiers for bankruptcy prediction and credit scoring}},
url = {http://dx.doi.org/10.1016/j.eswa.2008.01.018},
volume = {36},
year = {2009}
}
@article{Dong2011,
author = {Dong, Yanjie and Wang, Xuehua},
doi = {10.1007/978-3-642-25975-3\_30},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Dong, Wang - 2011 - A new over-sampling approach Random-SMOTE for learning from imbalanced data sets.pdf:pdf},
isbn = {9783642259746},
issn = {03029743},
journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
keywords = {Imbalanced Data sets,Over-sampling Approach,Random-SMOTE},
pages = {343--352},
title = {{A new over-sampling approach: Random-SMOTE for learning from imbalanced data sets}},
volume = {7091 LNAI},
year = {2011}
}
@article{Vanderlooy2008,
abstract = {The area under the ROC curve, or AUC, has been widely used to assess the ranking performance of binary scoring classifiers. Given a sample, the metric considers the ordering of positive and negative instances, i.e., the sign of the corresponding score differences. From a model evaluation and selection point of view, it may appear unreasonable to ignore the absolute value of these differences. For this reason, several variants of the AUC metric that take score differences into account have recently been proposed. In this paper, we present a unified framework for these metrics and provide a formal analysis. We conjecture that, despite their intuitive appeal, actually none of the variants is effective, at least with regard to model evaluation and selection. An extensive empirical analysis corroborates this conjecture. Our findings also shed light on recent research dealing with the construction of AUC-optimizing classifiers.},
author = {Vanderlooy, Stijn and H\"{u}llermeier, Eyke},
doi = {10.1007/s10994-008-5070-x},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Vanderlooy, H\"{u}llermeier - 2008 - A critical analysis of variants of the AUC.pdf:pdf},
isbn = {9783540874782},
issn = {08856125},
journal = {Machine Learning},
keywords = {AUC maximization,AUC variants,Area under the ROC curve,Bias-variance analysis,ROC analysis,Ranking performance},
number = {3},
pages = {247--262},
title = {{A critical analysis of variants of the AUC}},
volume = {72},
year = {2008}
}
@book{Wasserman2004,
address = {New York, NY},
author = {Wasserman, Larry},
doi = {10.1007/978-0-387-21736-9},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wasserman - 2004 - All of Statistics.pdf:pdf},
isbn = {978-1-4419-2322-6},
publisher = {Springer New York},
series = {Springer Texts in Statistics},
title = {{All of Statistics}},
url = {http://link.springer.com/10.1007/978-0-387-21736-9},
year = {2004}
}
@book{Robert2004a,
author = {Robert, Christian P. and Casella, George},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Robert, Casella - 2004 - Indroducing Monte Carlo Methods with R.pdf:pdf},
isbn = {9781441915757},
keywords = {Monte Carlo,R,available in PDF,simulation,textbook},
title = {{Indroducing Monte Carlo Methods with R}},
year = {2004}
}
@article{Weinberger2009b,
abstract = {Empirical evidence suggests that hashing is an effective strategy for dimensionality reduction and practical nonparametric estimation. In this paper we provide exponential tail bounds for feature hashing and show that the interaction between random subspaces is negligible with high probability. We demonstrate the feasibility of this approach with experimental results for a new use case -- multitask learning with hundreds of thousands of tasks.},
archivePrefix = {arXiv},
arxivId = {0902.2206},
author = {Weinberger, Kilian and Dasgupta, Anirban and Attenberg, Josh and Langford, John and Smola, Alex},
doi = {10.1145/1553374.1553516},
eprint = {0902.2206},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Weinberger et al. - 2009 - Feature Hashing for Large Scale Multitask Learning.pdf:pdf},
isbn = {9781605585161},
issn = {1605585165},
title = {{Feature Hashing for Large Scale Multitask Learning}},
url = {http://arxiv.org/abs/0902.2206},
year = {2009}
}
@article{Chen,
author = {Chen, Shaobing},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Chen - Unknown - Basis Pursuit 2 Goals of Adaptive Representation 3 Finding an Adaptive Representation.pdf:pdf},
journal = {Carbon},
title = {{Basis Pursuit 2 Goals of Adaptive Representation 3 Finding an Adaptive Representation}}
}
@article{Khoshgoftaar2007,
abstract = {This paper discusses a comprehensive suite of experiments that analyze the performance of the random forest (RF) learner implemented in Weka. RF is a relatively new learner, and to the best of our knowledge, only preliminary experimentation on the construction of random forest classifiers in the context of imbalanced data has been reported in previous work. Therefore, the contribution of this study is to provide an extensive empirical evaluation of RF learners built from imbalanced data. What should be the recommended default number of trees in the ensemble? What should the recommended value be for the number of attributes? How does the RF learner perform on imbalanced data when compared with other commonly-used learners? We address these and other related issues in this work.},
author = {Khoshgoftaar, T.M. and Golawala, M. and Hulse, J. Van},
doi = {10.1109/ICTAI.2007.46},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Khoshgoftaar, Golawala, Hulse - 2007 - An Empirical Study of Learning from Imbalanced Data Using Random Forest.pdf:pdf},
isbn = {978-0-7695-3015-4},
issn = {1082-3409},
journal = {19th IEEE International Conference on Tools with Artificial Intelligence(ICTAI 2007)},
pages = {310--317},
title = {{An Empirical Study of Learning from Imbalanced Data Using Random Forest}},
volume = {2},
year = {2007}
}
@article{He2008,
abstract = {This paper presents a novel adaptive synthetic (ADASYN) sampling approach for learning from imbalanced data sets. The essential idea of ADASYN is to use a weighted distribution for different minority class examples according to their level of difficulty in learning, where more synthetic data is generated for minority class examples that are harder to learn compared to those minority examples that are easier to learn. As a result, the ADASYN approach improves learning with respect to the data distributions in two ways: (1) reducing the bias introduced by the class imbalance, and (2) adaptively shifting the classification decision boundary toward the difficult examples. Simulation analyses on several machine learning data sets show the effectiveness of this method across five evaluation metrics.},
author = {He, Haibo and Bai, Yang and Garcia, Edwardo a. and Li, Shutao},
doi = {10.1109/IJCNN.2008.4633969},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/He et al. - 2008 - ADASYN Adaptive synthetic sampling approach for imbalanced learning.pdf:pdf},
isbn = {9781424418213},
issn = {1098-7576},
journal = {Proceedings of the International Joint Conference on Neural Networks},
number = {3},
pages = {1322--1328},
title = {{ADASYN: Adaptive synthetic sampling approach for imbalanced learning}},
year = {2008}
}
@article{Cao2009,
abstract = {Credit risk models are used by financial companies to evaluate in advance the insolvency risk caused by credits that enter into default. Many models for credit risk have been developed over the past few decades. In this paper, we focus on those models that can be formulated in terms of the probability of default by using survival analysis techniques. With this objective three different mechanisms are proposed based on the key idea of writing the default probability in terms of the conditional distribution function of the time to default. The first method is based on a Cox's regression model, the second approach uses generalized linear models under censoring and the third one is based on nonparametric kernel estimation, using the product-limit conditional distribution function estimator by Beran. The resulting nonparametric estimator of the default probability is proved to be consistent and asymptotically normal. An empirical study, based on modified real data, illustrates the three methods.},
author = {Cao, Ricardo and Vilar, Juan M. and Devia, Andr\'{e}s},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cao, Vilar, Devia - 2009 - Modelling consumer credit risk via survival analysis.pdf:pdf},
isbn = {1696-2281},
issn = {16962281},
journal = {SORT Statistics and Operations Research Transactions},
keywords = {basel ii,conditional survival function,generalized product-limit estimator,nonparametric regression,probability of default,q\"{u}estii\'{o},sort},
number = {June},
pages = {1},
title = {{Modelling consumer credit risk via survival analysis}},
volume = {33},
year = {2009}
}
@article{Gall2010,
author = {Gall, Jean-fran\c{c}ois Le},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Gall - 2010 - Random trees.pdf:pdf},
number = {August},
pages = {1--40},
title = {{Random trees}},
year = {2010}
}
@book{Tsybakov2009,
author = {Tsybakov, A and Zaiats, V},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Tsybakov, Zaiats - 2009 - Introduction to nonparametric estimation.pdf:pdf},
isbn = {9780387790510},
title = {{Introduction to nonparametric estimation}},
url = {http://link.springer.com/content/pdf/10.1007/b13794.pdf},
year = {2009}
}
@article{Hastie2000,
abstract = {We propose general procedures for posterior sampling from additive and generalized additive models. The procedure is a stochastic generalization of the well-known backfitting algorithm for fitting additive models. One chooses a linear operator ("smoother") for each predictor, and the algorithm requires only the application of the operator and its square root. The procedure is general and modular, and we describe its application to nonparametric, semiparametric and mixed models.},
author = {Hastie, T and Tibshirani, R},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hastie, Tibshirani - 2000 - Bayesian backfitting.pdf:pdf},
isbn = {0883-4237},
issn = {0883-4237},
journal = {Statistical Science},
keywords = {additive models,backfitting,bayes,distributions,gibbs,gibbs sampling,markov-chains,metropolis-hastings procedure,random effects,state-space models},
number = {3},
pages = {196--213},
title = {{Bayesian backfitting}},
url = {<Go to ISI>://000166404100002},
volume = {15},
year = {2000}
}
@article{Schulter2013,
abstract = {This paper introduces a novel classification method termed Alternating Decision Forests (ADFs), which formulates the training of Random Forests explicitly as a global loss minimization problem. During training, the losses are minimized via keeping an adaptive weight distribution over the training samples, similar to Boosting methods. In order to keep the method as flexible and general as possible, we adopt the principle of employing gradient descent in function space, which allows to minimize arbitrary losses. Contrary to Boosted Trees, in our method the loss minimization is an inherent part of the tree growing process, thus allowing to keep the benefits of common Random Forests, such as, parallel processing. We derive the new classifier and give a discussion and evaluation on standard machine learning data sets. Furthermore, we show how ADFs can be easily integrated into an object detection application. Compared to both, standard Random Forests and Boosted Trees, ADFs give better performance in our experiments, while yielding more compact models in terms of tree depth.},
author = {Schulter, Samuel and Wohlhart, Paul and Leistner, Christian and Saffari, Amir and Roth, Peter M. and Bischof, Horst},
doi = {10.1109/CVPR.2013.72},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Schulter et al. - 2013 - Alternating decision forests.pdf:pdf},
isbn = {978-0-7695-4989-7},
issn = {10636919},
journal = {Proceedings of the IEEE Computer Society Conference on Computer  Vision and Pattern Recognition},
keywords = {Boosting,Global Loss,Random Forests},
pages = {508--515},
title = {{Alternating decision forests}},
year = {2013}
}
@article{Baker2015,
abstract = {Searches for gravitational waves produced by coalescing black hole binaries with total masses ≳25 M⊙ use matched filtering with templates of short duration. Non-Gaussian noise bursts in gravitational wave detector data can mimic short signals and limit the sensitivity of these searches. Previous searches have relied on empirically designed statistics incorporating signal-to-noise ratio and signal-based vetoes to separate gravitational wave candidates from noise candidates. We report on sensitivity improvements achieved using a multivariate candidate ranking statistic derived from a supervised machine learning algorithm. We apply the random forest of bagged decision trees technique to two separate searches in the high mass (≳25 M⊙) parameter space. For a search which is sensitive to gravitational waves from the inspiral, merger, and ringdown of binary black holes with total mass between 25 M⊙ and 100 M⊙, we find sensitive volume improvements as high as 70±13\%–109±11\% when compared to the previously used ranking statistic. For a ringdown-only search which is sensitive to gravitational waves from the resultant perturbed intermediate mass black hole with mass roughly between 10 M⊙ and 600 M⊙, we find sensitive volume improvements as high as 61±4\%–241±12\% when compared to the previously used ranking statistic. We also report how sensitivity improvements can differ depending on mass regime, mass ratio, and available data quality information. Finally, we describe the techniques used to tune and train the random forest classifier that can be generalized to its use in other searches for gravitational waves.},
archivePrefix = {arXiv},
arxivId = {1412.6479v1},
author = {Baker, Paul T and Caudill, Sarah and Hodge, Kari A and Talukder, Dipongkar and Capano, Collin and Cornish, Neil J},
doi = {10.1103/PhysRevD.91.062004},
eprint = {1412.6479v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Baker et al. - 2015 - Multivariate classification with random forests for gravitational wave searches of black hole binary coalescence.pdf:pdf},
issn = {1550-7998},
journal = {Physical Review D},
number = {6},
pages = {1--27},
title = {{Multivariate classification with random forests for gravitational wave searches of black hole binary coalescence}},
url = {http://link.aps.org/doi/10.1103/PhysRevD.91.062004},
volume = {91},
year = {2015}
}
@article{Biau2010,
abstract = {Let X1...,Xn be identically distributed random vectors in Rd, independently drawn according to some probability density. An observation Xi is said to be a layered nearest neighbour (LNN) of a point x if the hyperrectangle defined by x and Xi contains no other data points. We first establish consistency results on Ln(x), the number of LNN of x. Then, given a sample (X,Y),(X1,Y1)...,(Xn,Yn) of independent identically distributed random vectors from Rd, one may estimate the regression function (x)=E[Y|X=x] by the LNN estimate rn(x), defined as an average over the Yi's corresponding to those Xi which are LNN of x. Under mild conditions on r, we establish the consistency of E|rn(x)-r(x)|p towards 0 as n→∞E;, for almost all x and all p≥1, and discuss the links between rn and the random forest estimates of Breiman (2001) [8]. We finally show the universal consistency of the bagged (bootstrap-aggregated) nearest neighbour method for regression and classification. © 2010 Elsevier Inc.},
annote = {Theorie Interessante},
author = {Biau, G\'{e}rard and Devroye, Luc},
doi = {10.1016/j.jmva.2010.06.019},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Biau, Devroye - 2010 - On the layered nearest neighbour estimate, the bagged nearest neighbour estimate and the random forest method in.pdf:pdf},
issn = {0047259X},
journal = {Journal of Multivariate Analysis},
keywords = {Bagging,Layered nearest neighbours,One nearest neighbour estimate,Random forests,Regression estimation},
number = {10},
pages = {2499--2518},
title = {{On the layered nearest neighbour estimate, the bagged nearest neighbour estimate and the random forest method in regression and classification}},
volume = {101},
year = {2010}
}
@article{Hutter2011,
abstract = {State-of-the-art algorithms for hard computational problems often expose many parameters that can be modified to improve empirical performance. However, manually exploring the resulting combinatorial space of parameter settings is tedious and tends to lead to unsatisfactory outcomes. Recently, automated approaches for solving this algorithm configuration problem have led to substantial improvements in the state of the art for solving various problems. One promising approach constructs explicit regression models to describe the dependence of target algorithm performance on parameter settings; however, this approach has so far been limited to the optimization of few numerical algorithm parameters on single instances. In this paper, we extend this paradigm for the first time to general algorithm configuration problems, allowing many categorical parameters and optimization for sets of instances. We experimentally validate our new algorithm configuration procedure by optimizing a local search and a tree search solver for the propositional satisfiability problem (\{SAT)\}, as well as the commercial mixed integer programming (\{MIP)\} solver \{CPLEX.\} In these experiments, our procedure yielded state-of-the-art performance, and in many cases outperformed the previous best configuration approach.},
author = {Hutter, Frank and Hoos, Holger H. and Leyton-Brown, Kevin},
doi = {10.1007/978-3-642-25566-3\_40},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hutter, Hoos, Leyton-Brown - 2011 - Sequential model-based optimization for general algorithm configuration.pdf:pdf},
isbn = {9783642255656},
issn = {03029743},
journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
pages = {507--523},
title = {{Sequential model-based optimization for general algorithm configuration}},
volume = {6683 LNCS},
year = {2011}
}
@book{Breiman1984a,
abstract = {The methodology used to construct tree structured rules is the focus of this monograph. Unlike many other statistical procedures, which moved from pencil and paper to calculators, this text's use of trees was unthinkable before computers. Both the practical and theoretical sides have been developed in the authors' study of tree methods. Classification and Regression Trees reflects these two sides, covering the use of trees as a data analysis method, and in a more mathematical framework, proving some of their fundamental properties.},
author = {Breiman, Leo and Friedman, Jerome H and Olshen, R A and Stone, C J},
booktitle = {Wadsworth International Group},
isbn = {0412048418},
pages = {368},
title = {{Classification and Regression Trees}},
url = {http://scholar.google.com/scholar?hl=en\&btnG=Search\&q=intitle:Classification+and+regression+trees\#0},
volume = {p},
year = {1984}
}
@article{Cortes1995,
author = {Cortes, Corinna and Vapnik, Vladimir},
doi = {10.1007/BF00994018},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cortes, Vapnik - 1995 - Support-vector networks.pdf:pdf},
issn = {0885-6125},
journal = {Machine Learning},
keywords = {SVM,VC dimension,margin},
mendeley-tags = {SVM,VC dimension,margin},
month = sep,
number = {3},
pages = {273--297},
title = {{Support-vector networks}},
url = {http://link.springer.com/10.1007/BF00994018},
volume = {20},
year = {1995}
}
@phdthesis{Comminges2012a,
abstract = {Les donn\'{e}es du monde r\'{e}el sont souvent de tr\`{e}s grande dimension, faisant intervenir un grand nombre de variables non pertinentes ou redondantes. La s\'{e}lection de variables est donc utile dans ce cadre. D’abord, on consid\`{e}re la s\'{e}lection de variables dans le mod\`{e}le de r\'{e}gression quand le nombre de variables est tr\`{e}s grand. En particulier on traite le cas o\`{u} le nombre de variables pertinentes est bien plus petit que la dimension ambiante. Sans sup- poser aucune forme param\'{e}trique pour la fonction de r\'{e}gression, on obtient des conditions minimales permettant de retrouver l’ensemble des variables pertinentes. Ces conditions relient la dimension intrins\`{e}que \`{a} la dimension ambiante et la taille de l’\'{e}chantillon. En- suite, on consid\`{e}re le probl\`{e}me du test d’une hypoth\`{e}se nulle composite sous un mod\`{e}le de r\'{e}gression nonparam\'{e}trique multivari\'{e}. Pour une fonctionnelle quadratique donn\'{e}e Q, l’hypoth\`{e}se nulle correspond au fait que la fonction f satisfait la contrainte Q[f] = 0, tandis que l’alternative correspond aux fonctions pour lesquelles |Q[f]| est minor\'{e}e par une constante strictement positive. On fournit des taux minimax de test et les constantes de s\'{e}paration exactes ainsi qu’une proc\'{e}dure optimale exacte, pour des fonctionnelles qua- dratiques diagonales et positives. On peut utiliser ces r\'{e}sultats pour tester la pertinence d’une ou plusieurs variables explicatives. L’\'{e}tude des taux minimax pour les fonctionnelles quadratiques diagonales qui ne sont ni positives ni n\'{e}gatives, fait appara\^{\i}tre deux r\'{e}gimes diff\'{e}rents : un r\'{e}gime «r\'{e}gulier» et un r\'{e}gime «irr\'{e}gulier». On applique ceci au test de l’\'{e}galit\'{e}},
author = {Comminges, La\"{e}titia},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Comminges - 2012 - Comminges Quelques contributions \`{a} la s\'{e}lection de variables.pdf:pdf},
title = {{Comminges Quelques contributions \`{a} la s\'{e}lection de variables}},
year = {2012}
}
@article{Amaratunga2008,
abstract = {Although the random forest classification procedure works well in datasets with many features, when the number of features is huge and the percentage of truly informative features is small, such as with DNA microarray data, its performance tends to decline significantly. In such instances, the procedure can be improved by reducing the contribution of trees whose nodes are populated by non-informative features. To some extent, this can be achieved by prefiltering, but we propose a novel, yet simple, adjustment that has demonstrably superior performance: choose the eligible subsets at each node by weighted random sampling instead of simple random sampling, with the weights tilted in favor of the informative features. This results in an 'enriched random forest'. We illustrate the superior performance of this procedure in several actual microarray datasets.},
author = {Amaratunga, Dhammika and Cabrera, Javier and Lee, Yung Seop},
doi = {10.1093/bioinformatics/btn356},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Amaratunga, Cabrera, Lee - 2008 - Enriched random forests.pdf:pdf},
isbn = {1367-4811 (Electronic)$\backslash$r1367-4803 (Linking)},
issn = {13674803},
journal = {Bioinformatics},
number = {18},
pages = {2010--2014},
pmid = {18650208},
title = {{Enriched random forests}},
volume = {24},
year = {2008}
}
@article{Vilalta2002,
abstract = {Different researchers hold different views of what the term meta-learning exactlymeans. The first part of this paper provides our own perspective view in which the goal isto build self-adaptive learners (i.e. learning algorithms that improve their bias dynamicallythrough experience by accumulating meta-knowledge). The second part provides a survey ofmeta-learning as reported by the machine-learning literature. We find that, despite differentviews and research lines, a question remains constant: how can we exploit knowledge aboutlearning (i.e. meta-knowledge) to improve the performance of learning algorithms? Clearlythe answer to this question is key to the advancement of the field and continues being thesubject of intensive research.},
author = {Vilalta, Ricardo and Drissi, Youssef},
doi = {10.1023/A:1019956318069},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Vilalta, Drissi - 2002 - A perspective view and survey of meta-learning.pdf:pdf},
issn = {02692821},
journal = {Artificial Intelligence Review},
keywords = {Classification,Inductive learning,Meta-knowledge},
number = {2},
pages = {77--95},
title = {{A perspective view and survey of meta-learning}},
volume = {18},
year = {2002}
}
@book{Højsgaard2012a,
abstract = {Graphical models in their modern form have been around since the late 1970s and appear today in many areas of the sciences. Along with the ongoing developments of graphical models, a number of different graphical modeling software programs have been written over the years. In recent years many of these software developments have taken place within the R community, either in the form of new packages or by providing an R interface to existing software. This book attempts to give the reader a gentle introduction to graphical modeling using R and the main features of some of these packages. In addition, the book provides examples of how more advanced aspects of graphical modeling can be represented and handled within R. Topics covered in the seven chapters include graphical models for contingency tables, Gaussian and mixed graphical models, Bayesian networks and modeling high dimensional data.},
author = {H\o jsgaard, S\o ren and Edwards, David and Lauritzen, Steffen},
doi = {10.1007/978-1-4614-2299-0},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/H\o jsgaard, Edwards, Lauritzen - 2012 - Graphical Models with R.pdf:pdf},
isbn = {1461422981},
pages = {1--182},
title = {{Graphical Models with R}},
url = {http://www.amazon.com/Graphical-Models-R-Use/dp/1461422981/ref=sr\_1\_1?s=books\&ie=UTF8\&qid=1402070081\&sr=1-1\&keywords=graphical+models+with+R},
year = {2012}
}
@techreport{J,
author = {Friedman, Jerome H},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Friedman - 1999 - Greedy Function Approximation A Gradient Boosting Machine.pdf:pdf},
title = {{Greedy Function Approximation: A Gradient Boosting Machine}},
year = {1999}
}
@article{Baszczynski2013,
author = {Błaszczyński, Jerzy and Stefanowski, Jerzy and Idkowiak, Łukasz},
doi = {10.1007/978-3-319-00969-8\_26},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Błaszczyński, Stefanowski, Idkowiak - 2013 - Extending bagging for imbalanced data.pdf:pdf},
isbn = {9783319009681},
issn = {21945357},
journal = {Advances in Intelligent Systems and Computing},
pages = {269--278},
title = {{Extending bagging for imbalanced data}},
volume = {226},
year = {2013}
}
@article{Caruana2008,
author = {Caruana, Rich and Karampatziakis, Nikos and Yessenalina, Ainur},
doi = {10.1145/1390156.1390169},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Caruana, Karampatziakis, Yessenalina - 2008 - An empirical evaluation of supervised learning in high dimensions.pdf:pdf},
isbn = {9781605582054},
journal = {Proceedings of the 25th international conference on Machine learning - ICML '08},
pages = {96--103},
title = {{An empirical evaluation of supervised learning in high dimensions}},
url = {http://portal.acm.org/citation.cfm?doid=1390156.1390169},
year = {2008}
}
@article{Hsieh2010,
abstract = {This study focuses on predicting whether a credit applicant can be categorized as good, bad or borderline from information initially supplied. This is essentially a classification task for credit scoring. Given its importance, many researchers have recently worked on an ensemble of classifiers. However, to the best of our knowledge, unrepresentative samples drastically reduce the accuracy of the deployment classifier. Few have attempted to preprocess the input samples into more homogeneous cluster groups and then fit the ensemble classifier accordingly. For this reason, we introduce the concept of class-wise classification as a preprocessing step in order to obtain an efficient ensemble classifier. This strategy would work better than a direct ensemble of classifiers without the preprocessing step. The proposed ensemble classifier is constructed by incorporating several data mining techniques, mainly involving optimal associate binning to discretize continuous values; neural network, support vector machine, and Bayesian network are used to augment the ensemble classifier. In particular, the Markov blanket concept of Bayesian network allows for a natural form of feature selection, which provides a basis for mining association rules. The learned knowledge is represented in multiple forms, including causal diagram and constrained association rules. The data driven nature of the proposed system distinguishes it from existing hybrid/ensemble credit scoring systems. ?? 2009 Elsevier Ltd. All rights reserved.},
author = {Hsieh, Nan Chen and Hung, Lun Ping},
doi = {10.1016/j.eswa.2009.05.059},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hsieh, Hung - 2010 - A data driven ensemble classifier for credit scoring analysis.pdf:pdf},
isbn = {3642013066},
issn = {09574174},
journal = {Expert Systems with Applications},
keywords = {Bayesian network,Class-wise classification,Clustering,Credit scoring system,Ensemble classifier,Neural network},
number = {1},
pages = {534--545},
publisher = {Elsevier Ltd},
title = {{A data driven ensemble classifier for credit scoring analysis}},
url = {http://dx.doi.org/10.1016/j.eswa.2009.05.059},
volume = {37},
year = {2010}
}
@article{Bae2009,
author = {Bae, Choongsoon},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bae - 2009 - Versions of Random Forests Properties and Performances.pdf:pdf},
journal = {Nature},
title = {{Versions of Random Forests : Properties and Performances}},
year = {2009}
}
@inproceedings{Bostrom2008,
abstract = {When using the output of classifiers to calculate the expected utility of different alternatives in decision situations, the correctness of predicted class probabilities may be of crucial importance. However, even very accurate classifiers may output class probabilities of rather poor quality. One way of overcoming this problem is by means of calibration, i.e., mapping the original class probabilities to more accurate ones. Previous studies have however indicated that random forests are difficult to calibrate by standard calibration methods. In this work, a novel calibration method is introduced, which is based on a recent finding that probabilities predicted by forests of classification trees have a lower squared error compared to those predicted by forests of probability estimation trees (PETs). The novel calibration method is compared to the two standard methods, Platt scaling and isotonic regression, on 34 datasets from the UCI repository. The experiment shows that random forests of PETs calibrated by the novel method significantly outperform uncalibrated random forests of both PETs and classification trees, as well as random forests calibrated with the two standard methods, with respect to the squared error of predicted class probabilities.},
author = {Bostr\"{o}m, Henrik},
booktitle = {Proceedings - 7th International Conference on Machine Learning and Applications, ICMLA 2008},
doi = {10.1109/ICMLA.2008.107},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bostr\"{o}m - 2008 - Calibrating random forests.pdf:pdf},
isbn = {9780769534954},
pages = {121--126},
title = {{Calibrating random forests}},
year = {2008}
}
@article{Schapire2003a,
abstract = {Boosting is a general method for improving the accuracy of any given learning algorithm. Focusing primarily on the AdaBoost algorithm, this chapter overviews some of the recent work on boosting including analyses of AdaBoost’s training error and generalization error; boosting’s connection to game theory and linear programming; the relationship between boosting and logistic regression; extensions of AdaBoost for multiclass classification problems; methods of incorporating human knowledge into boosting; and experimental and applied work using boosting.},
author = {Schapire, Robert E},
doi = {10.1.1.24.5565},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Schapire - 2003 - The boosting approach to machine learning an overview.pdf:pdf},
isbn = {978-0-387-95471-4},
issn = {09300325},
journal = {Nonlinear Estimation and Classification},
pages = {149--171},
title = {{The boosting approach to machine learning: an overview}},
url = {http://www.ams.org/mathscinet/search/publications.html?pg1=MR\&s1=MR2005788$\backslash$npapers2://publication/uuid/13F19159-186B-4FAA-9433-AE2BE3D153D8},
volume = {171},
year = {2003}
}
@article{Jones1998a,
abstract = {In many engineering optimization problems, the number of function evaluations is severely limited by time or cost. These problems pose a special challenge to the field of global optimization, since existing methods often require more function evaluations than can be comfortably afforded. One way to address this challenge is to fit response surfaces to data collected by evaluating the objective and constraint functions at a few points. These surfaces can then be used for visualization, tradeoff analysis, and optimization. In this paper, we introduce the reader to a response surface methodology that is especially good at modeling the nonlinear, multimodal functions that often occur in engineering. We then show how these approximating functions can be used to construct an efficient global optimization algorithm with a credible stopping rule. The key to using response surfaces for global optimization lies in balancing the need to exploit the approximating surface (by sampling where it is minimized) with the need to improve the approximation (by sampling where prediction error may be high). Striking this balance requires solving certain auxiliary problems which have previously been considered intractable, but we show how these computational obstacles can be overcome.},
author = {Jones, Dr and Schonlau, Matthias and Welch, Wj},
doi = {10.1023/a:1008306431147},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Jones, Schonlau, Welch - 1998 - Efficient global optimization of expensive black-box functions.pdf:pdf},
isbn = {0925-5001},
issn = {09255001},
journal = {Journal of Global optimization},
keywords = {bayesian global optimization,kriging,process,random function,response surface,stochastic,visualization},
number = {4},
pages = {455--492},
pmid = {21858987},
title = {{Efficient global optimization of expensive black-box functions}},
url = {http://www.springerlink.com/index/M5878111M101017P.pdf$\backslash$nhttp://link.springer.com/article/10.1023/A:1008306431147},
volume = {13},
year = {1998}
}
@article{Yap2011a,
abstract = {Credit scoring model have been developed by banks and researchers to improve the process of assessing credit worthiness during the credit evaluation process. The objective of credit scoring models is to assign credit risk to either a "good risk" group that is likely to repay financial obligation or a "bad risk" group who has high possibility of defaulting on the financial obligation. Construction of credit scoring models requires data mining techniques. Using historical data on payments, demographic characteristics and statistical techniques, credit scoring models can help identify the important demographic characteristics related to credit risk and provide a score for each customer. This paper illustrates using data mining to improve assessment of credit worthiness using credit scoring models. Due to privacy concerns and unavailability of real financial data from banks this study applies the credit scoring techniques using data of payment history of members from a recreational club. The club has been facing a problem of rising number in defaulters in their monthly club subscription payments. The management would like to have a model which they can deploy to identify potential defaulters. The classification performance of credit scorecard model, logistic regression model and decision tree model were compared. The classification error rates for credit scorecard model, logistic regression and decision tree were 27.9\%, 28.8\% and 28.1\%, respectively. Although no model outperforms the other, scorecards are relatively much easier to deploy in practical applications. ?? 2010 Elsevier Ltd. All rights reserved.},
author = {Yap, Bee Wah and Ong, Seng Huat and Husain, Nor Huselina Mohamed},
doi = {10.1016/j.eswa.2011.04.147},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Yap, Ong, Husain - 2011 - Using data mining to improve assessment of credit worthiness via credit scoring models.pdf:pdf},
isbn = {978-1-61284-108-3},
issn = {09574174},
journal = {Expert Systems with Applications},
keywords = {Classification,Credit scoring,Data mining,Decision tree,Logistic regression,Predictive modeling},
number = {10},
pages = {13274--13283},
publisher = {Elsevier Ltd},
title = {{Using data mining to improve assessment of credit worthiness via credit scoring models}},
url = {http://dx.doi.org/10.1016/j.eswa.2011.04.147},
volume = {38},
year = {2011}
}
@article{Utgoff1989,
abstract = {This article presents an incremental algorithm for inducing decision trees equivalent to those formed by Quinlan's nonincremental ID3 algorithm, given the same training instances. The new algorithm, named ID5R, lets one apply the ID3 induction process to learning tasks in which training instances are presented serially.},
author = {Utgoff, Paul E.},
doi = {10.1.1.10.181},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Utgoff - 1989 - Incremental Induction of Decision Trees.pdf:pdf},
issn = {0885-6125},
journal = {Machine Learning},
keywords = {concept learning,decision tree,incremental learning,learning from examples},
number = {2},
pages = {161--186},
title = {{Incremental Induction of Decision Trees}},
url = {http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.10.181},
volume = {4},
year = {1989}
}
@article{Geurts2006,
abstract = {This paper proposes a newtree-based ensemblemethod for supervised classification and regression problems. It essentially consists of randomizing strongly both attribute and cut-point choice while splitting a tree node. In the extreme case, it builds totally randomized trees whose structures are independent of the output values of the learning sample. The strength of the randomization can be tuned to problem specifics by the appropriate choice of a parameter. We evaluate the robustness of the default choice of this parameter, and we also provide insight on how to adjust it in particular situations. Besides accuracy, the main strength of the resulting algorithm is computational efficiency.Abias/variance analysis of the Extra-Trees algorithm is also provided as well as a geometrical and a kernel characterization of the models induced. Keywords},
author = {Geurts, Pierre and Ernst, Damien and Wehenkel, Louis},
doi = {10.1007/s10994-006-6226-1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Geurts, Ernst, Wehenkel - 2006 - Extremely randomized trees.pdf:pdf},
isbn = {0885-6125},
issn = {08856125},
journal = {Machine Learning},
keywords = {Bias/variance tradeoff,Cut-point randomization,Decision and regression trees,Ensemble methods,Kernel-based models,Supervised learning},
number = {1},
pages = {3--42},
title = {{Extremely randomized trees}},
volume = {63},
year = {2006}
}
@article{Ramon2003,
abstract = {Bayesian classifiers such as Naive Bayes or Tree Augmented Naive Bayes (TAN) have shown excellent performance given their simplicity and heavy underlying independence assumptions. In this paper we introduce a classifier taking as basis the TAN model and taking into account uncertainty in model selection. To do this we introduce decomposable distributions over TANs and show that they allow the expression resulting from the Bayesian model averaging of TAN models to be integrated into closed form. With this result we construct a classifier with a shorter learning time and a longer classification time than TAN. Empirical results show that the classifier is, most of the cases, more accurate than TAN and approximates better the class probabilities.},
author = {Ramon, L\'{o}pez De M\`{a}ntaras and Cerquides, Jes\'{u}s},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ramon, Cerquides - 2003 - Tractable Bayesian Learning of Tree Augmented Naive Bayes Models.pdf:pdf},
isbn = {1577351894},
journal = {Proceedings of the Twentieth International Conference on Machine Learning (ICML-2003)},
pages = {1--25},
title = {{Tractable Bayesian Learning of Tree Augmented Naive Bayes Models}},
volume = {1},
year = {2003}
}
@book{Brezis2011,
author = {Brezis, H},
doi = {10.1007/978-0-387-70914-7},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Brezis - 2011 - Functional analysis, Sobolev spaces and partial differential equations.pdf:pdf},
isbn = {9780387709130},
title = {{Functional analysis, Sobolev spaces and partial differential equations}},
url = {http://link.springer.com/content/pdf/10.1007/978-0-387-70914-7.pdf},
year = {2011}
}
@article{Zhai2011,
abstract = {Imbalanced data sets in real-world applications have a majority class with normal instances and a minority class with abnormal or important instances. Learning from such data sets usually generates biased classifiers that have a higher predictive accuracy over the majority class, but a rather poorer predictive accuracy over the minority class. The Synthetic minority over-sampling technique (SMOTE) is specifically designed for learning from imbalanced data sets. This paper presents a novel approach for learning from imbalanced data sets, based on an improved SMOTE algorithm. The approach deals with noise data by a hierarchical filtering mechanism, employs a selection strategy of the minority instances and makes full use of dynamic distribution density of the minority followed by the SMOTE process. This empirical analysis of the approach showed quantitatively competitive with SMOTE and series of its improved algorithm in terms of the receiver operating characteristic curve when applied to several highly and moderately imbalanced data sets.},
author = {Zhai, Yun and Ma, Nan and An, Bing and Ruan, Da},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Zhai et al. - 2011 - An effective over-sampling method for imbalanced data sets classification.pdf:pdf},
issn = {10224653},
journal = {Chinese Journal of Electronics},
keywords = {Classification,Data mining,Distribution density,Imbalanced data sets,Over-sample,Selection strategy},
number = {3},
pages = {489--494},
title = {{An effective over-sampling method for imbalanced data sets classification}},
volume = {20},
year = {2011}
}
@article{Panda2009,
abstract = {Classification and regression tree learning on massive datasets is a common data mining task at Google, yet many state of the art tree learning algorithms require training data to reside in memory on a single machine. While more scalable implementations of tree learning have been proposed, they typically require specialized parallel computing architectures. In contrast, the majority of Google's computing infrastructure is based on commodity hardware. In this paper, we describe PLANET: a scalable distributed framework for learning tree models over large datasets. PLANET defines tree learning as a series of distributed computations, and implements each one using the MapReduce model of distributed computation. We show how this framework supports scalable construction of classification and regression trees, as well as ensembles of such models. We discuss the benefits and challenges of using a MapReduce compute cluster for tree learning, and demonstrate the scalability of this approach by applying it to a real world learning task from the domain of computational advertising.},
author = {Panda, Biswanath and Herbach, Joshua S and Basu, Sugato and Bayardo, Roberto J},
doi = {10.14778/1687553.1687569},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Panda et al. - 2009 - PLANET massively parallel learning of tree ensembles with MapReduce.pdf:pdf},
isbn = {0000000000000},
issn = {21508097},
journal = {Proceedings of the VLDB Endowment},
number = {2},
pages = {1426--1437},
title = {{PLANET: massively parallel learning of tree ensembles with MapReduce}},
url = {http://portal.acm.org/citation.cfm?id=1687553.1687569},
volume = {2},
year = {2009}
}
@article{Approach2011,
abstract = {Research in combinatorial optimization initially focused on finding optimal solutions to various problems. Researchers realized the importance of alternative approaches when faced with large practical problems that took too long to solve optimally and this led to approaches like simulated annealing and genetic algorithms which could not guarantee optimality, but yielded good solutions within a reasonable amount of computing time. In this paper we report on our experiments with stochastic greedy algorithms (SGA) – perturbed versions of standard greedy algorithms. SGA incorporates the novel idea of learning from optimal solutions, inspired by data-mining and other learning approaches. SGA learns some characteristics of optimal solutions and then applies them while generating its solutions. We report results based on applying this approach to three different problems – knapsack, combinatorial auctions and single-machine job sequencing. Overall, the method consistently produces solutions significantly closer to optimal than standard greedy approaches. SGA can be seen in the space of approximate algorithms as falling between the very quick greedy approaches and the relatively slower soft computing approaches like genetic algorithms and simulated annealing. SGA is easy to understand and implement -- once a greedy solution approach is known for a problem, it becomes possible to very quickly rig up a SGA for the problem. SGA has explored only one aspect of learning from optimal solutions. We believe that there is a lot of scope for variations on the theme, and the broad idea of learning from optimal solutions opens},
author = {Viswanathan, Viswa and Sen, Anup K and Chakraborty, Soumyakanti},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Viswanathan, Sen, Chakraborty - 2011 - Stochastic Greedy Algorithms.pdf:pdf},
keywords = {- greedy algorithms,approximate solutions,auctions,combinatorial,knapsack problem,machine learning,single-machine scheduling,stochastic approaches},
number = {1},
pages = {1--11},
title = {{Stochastic Greedy Algorithms}},
volume = {4},
year = {2011}
}
@article{Chen2004,
abstract = {In this paper we propose two ways to deal with the imbalanced data classification problem using random forest. One is based on cost sensitive learning, and the other is based on a sampling technique. Performance metrics such as precision and recall, false positive rate and false negative rate, F-measure and weighted accuracy are computed. Both methods are shown to improve the prediction accuracy of the minority class, and have favorable performance compared to the existing algorithms. 1},
author = {Chen, Chao and Liaw, Andy and Breiman, Leo},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Chen, Liaw, Breiman - 2004 - Using random forest to learn imbalanced data.pdf:pdf},
journal = {University of California, Berkeley},
number = {1999},
pages = {1--12},
title = {{Using random forest to learn imbalanced data}},
year = {2004}
}
@article{Clemencon2013,
author = {Cl\'{e}men\c{c}on, St\'{e}phan and Depecker, Marine and Vayatis, Nicolas},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cl\'{e}men\c{c}on, Depecker, Vayatis - 2013 - Ranking Forests.pdf:pdf},
isbn = {1532-4435},
issn = {1532-4435},
journal = {Journal of Machine Learning Research},
keywords = {auc,bagging,bipartite ranking,bootstrap,classification data,criterion,feature,median ranking,nonparametric scoring,rank aggregation,roc optimization,tree-based ranking rules},
pages = {39--73},
title = {{Ranking Forests}},
volume = {14},
year = {2013}
}
@article{Achlioptas2003,
abstract = {A classic result of Johnson and Lindenstrauss asserts that any set of n points in d-dimensional Euclidean space can be embedded into k-dimensional Euclidean space - where k is logarithmic in n and independent of d - so that all pairwise distances are maintained within an arbitrarily small factor. All known constructions of such embeddings involve projecting the n points onto a spherically random k-dimensional hyperplane through the origin. We give two constructions of such embeddings with the property that all elements of the projection matrix belong in \{-1, 0, +1\}. Such constructions are particularly well suited for database environments, as the computation of the embedding reduces to evaluating a single aggregate over k random partitions of the attributes. ?? 2003 Elsevier Science (USA). All rights reserved.},
author = {Achlioptas, Dimitris},
doi = {10.1016/S0022-0000(03)00025-4},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Achlioptas - 2003 - Database-friendly random projections Johnson-Lindenstrauss with binary coins.pdf:pdf},
isbn = {1581133618},
issn = {00220000},
journal = {Journal of Computer and System Sciences},
number = {4},
pages = {671--687},
title = {{Database-friendly random projections: Johnson-Lindenstrauss with binary coins}},
volume = {66},
year = {2003}
}
@article{Bell2007,
abstract = {Our final solution (RMSE=0.8712) consists of blending 107 individual results. Since many of these results are close variants, we first describe the main approaches behind them. Then, we will move to describing each individual result. The core components of the solution are published in our ICDM'2007 paper 1 (or, KDD-Cup2007 paper 2), and also in the earlier KDD'2007 paper 3. We assume that the reader is familiar with these works and our terminology there.},
author = {Bell, Robert M and Koren, Yehuda and Volinsky, Chris},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bell, Koren, Volinsky - 2007 - The BellKor solution to the Netflix Prize.pdf:pdf},
isbn = {1862391149},
journal = {KorBell Teams Report to Netflix},
title = {{The BellKor solution to the Netflix Prize}},
url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.142.9009\&amp;rep=rep1\&amp;type=pdf},
volume = {2},
year = {2007}
}
@article{Smola2005,
abstract = {We present methods for dealing with missing variables in the context
  of Gaussian Processes and Support Vector Machines. This solves an
  important problem which has largely been ignored by kernel methods:
  How to systematically deal with incomplete data? Our method can also
  be applied to problems with partially observed labels as well as to
  the transductive setting where we view the labels as missing data.
  
  Our approach relies on casting kernel methods as an estimation
  problem in exponential families. Hence, estimation with missing
  variables becomes a problem of computing marginal distributions, and
  finding efficient optimization methods. To that extent we propose an
  optimization scheme which extends the Concave Convex Procedure (CCP)
  of Yuille and Rangarajan, and present a simplified and intuitive
  proof of its convergence. We show how our algorithm can be
  specialized to various cases in order to efficiently solve the
  optimization problems that arise. Encouraging preliminary
  experimental results on the USPS dataset are also presented.},
author = {Smola, Alex and Vishwanathan, S V N and Hoffman, Thomas},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Smola, Vishwanathan, Hoffman - 2005 - Kernel Methods for Missing Variables.pdf:pdf},
isbn = {097273581X},
keywords = {Learning/Statistics \& Optimisation,Theory \& Algorithms},
title = {{Kernel Methods for Missing Variables}},
url = {http://eprints.pascal-network.org/archive/00002053/},
year = {2005}
}
@article{Ishwaran2010a,
abstract = {The minimal depth of a maximal subtree is a dimensionless order statistic measuring the predictiveness of a variable in a survival tree.We derive the distribution of the minimal depth and use it for high-dimensional variable selection using random survival forests. In big p and small n problems (where p is the dimension and n is the sample size), the distribution of the minimal depth reveals a “ceiling effect” in which a tree simply cannot be grown deep enough to properly identify predictive variables.Motivated by this limitation, we develop a new regularized algorithm, termed RSF-Variable Hunting. This algorithm exploits maximal subtrees for effective variable selection under such scenarios. Several applications are presented demonstrating the methodology, including the problem of gene selection using microarray data. In this work we focus only on survival settings, although our methodology also applies to other random forests applications, including regression and classification settings. All examples presented here use the R-software package randomSurvivalForest.},
author = {Ishwaran, Hemant and Kogalur, Udaya B. and Gorodeski, Eiran Z. and Minn, Andy J. and Lauer, Michael S.},
doi = {10.1198/jasa.2009.tm08622},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ishwaran et al. - 2010 - High-Dimensional Variable Selection for Survival Data.pdf:pdf},
issn = {0162-1459},
journal = {Journal of the American Statistical Association},
keywords = {Forest, Maximal subtree, Minimal depth, Random sur,forest,maximal subtree,minimal depth,random survival forest,tree,vimp},
number = {489},
pages = {205--217},
title = {{High-Dimensional Variable Selection for Survival Data}},
volume = {105},
year = {2010}
}
@article{Gelbart2014,
abstract = {Recent work on Bayesian optimization has shown its effectiveness in global optimization of difficult black-box objective functions. Many real-world optimization problems of interest also have constraints which are unknown a priori. In this paper, we study Bayesian optimization for constrained problems in the general case that noise may be present in the constraint functions, and the objective and constraints may be evaluated independently. We provide motivating practical examples, and present a general framework to solve such problems. We demonstrate the effectiveness of our approach on optimizing the performance of online latent Dirichlet allocation subject to topic sparsity constraints, tuning a neural network given test-time memory constraints, and optimizing Hamiltonian Monte Carlo to achieve maximal effectiveness in a fixed time, subject to passing standard convergence diagnostics.},
archivePrefix = {arXiv},
arxivId = {arXiv:1403.5607v1},
author = {Gelbart, Michael A. and Snoek, Jasper and Adams, Ryan P.},
eprint = {arXiv:1403.5607v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Gelbart, Snoek, Adams - 2014 - Bayesian Optimization with Unknown Constraints.pdf:pdf},
isbn = {9780974903910},
journal = {arXiv},
keywords = {Computer Science - Learning,Statistics - Machine Learning},
pages = {1--14},
title = {{Bayesian Optimization with Unknown Constraints}},
url = {http://arxiv.org/abs/1403.5607},
year = {2014}
}
@article{Jahrer2010,
abstract = {We analyze the application of ensemble learning to recommender systems on the Netflix Prize dataset. For our analysis we use a set of diverse state-of-the-art collaborative filtering (CF) algorithms, which include: SVD, Neighborhood Based Approaches, Restricted Boltzmann Machine, Asymmetric Factor Model and Global Effects. We show that linearly combining (blending) a set of CF algorithms increases the accuracy and outperforms any single CF algorithm. Furthermore, we show how to use ensemble methods for blending predictors in order to outperform a single blending algorithm. The dataset and the source code for the ensemble blending are available online.},
author = {Jahrer, Michael and T\"{o}scher, Andreas and Legenstein, Robert},
doi = {10.1145/1835804.1835893},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Jahrer, T\"{o}scher, Legenstein - 2010 - Combining Predictions for an accurate Recommender System.pdf:pdf},
isbn = {9781450300551},
issn = {1450300553},
keywords = {Information Retrieval \& Textual Information Access,Theory \& Algorithms,User Modelling for Computer Human Interaction},
title = {{Combining Predictions for an accurate Recommender System}},
url = {http://eprints.pascal-network.org/archive/00006084/},
year = {2010}
}
@article{Garcia2012,
author = {Garc\'{\i}a, Vicente and Marqu\'{e}s, Ana Isabel and S\'{a}nchez, Jose Salvador},
doi = {10.1007/978-3-642-34481-7\_9},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Garc\'{\i}a, Marqu\'{e}s, S\'{a}nchez - 2012 - Improving risk predictions by preprocessing imbalanced credit data.pdf:pdf},
isbn = {9783642344800},
issn = {03029743},
journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
keywords = {Class imbalance,Classification,Credit scoring,Finance,Resampling},
number = {PART 2},
pages = {68--75},
title = {{Improving risk predictions by preprocessing imbalanced credit data}},
volume = {7664 LNCS},
year = {2012}
}
@book{Haykin2008,
author = {Haykin, Simon},
booktitle = {Pearson},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Haykin - 2008 - Neural Networks and Learning Machines.pdf:pdf},
isbn = {9780131471399},
keywords = {Machines Learning,Neural Network},
title = {{Neural Networks and Learning Machines}},
year = {2008}
}
@article{Zouari2005,
abstract = {In this paper, a simulation method is proposed to generate a set of classifier outputs with specified individual accuracies and fixed pairwise agreement. A diversity measure (kappa) is used to control the agreement among classifiers for building the classifier teams. The generated team outputs can be used to study the behaviour of class-type combination methods such as voting rules over multiple dependent classifiers. © 2004 Pattern Recognition Society. Published by Elsevier Ltd. All rights reserved.},
author = {Zouari, H\'{e}la and Heutte, Laurent and Lecourtier, Yves},
doi = {10.1016/j.patcog.2005.02.012},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Zouari, Heutte, Lecourtier - 2005 - Controlling the diversity in classifier ensembles through a measure of agreement.pdf:pdf},
issn = {00313203},
journal = {Pattern Recognition},
keywords = {Classifier agreement,Classifier ensemble,Classifier simulation,Dependency,Diversity measure,Kappa measure,Output generation algorithm},
number = {11},
pages = {2195--2199},
title = {{Controlling the diversity in classifier ensembles through a measure of agreement}},
volume = {38},
year = {2005}
}
@article{Candes2011,
author = {Cand\`{e}s, Emmanuel J. and Li, Xiaodong and Ma, Yi and Wright, John},
doi = {10.1145/1970392.1970395},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cand\`{e}s et al. - 2011 - Robust principal component analysis.pdf:pdf},
isbn = {0004-5411},
issn = {00045411},
journal = {Journal of the ACM},
number = {3},
pages = {1--37},
pmid = {19686071},
title = {{Robust principal component analysis?}},
volume = {58},
year = {2011}
}
@article{Tomita2015,
archivePrefix = {arXiv},
arxivId = {arXiv:1506.03410v1},
author = {Tomita, Tyler M and Maggioni, Mauro and Vogelstein, Joshua T},
eprint = {arXiv:1506.03410v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Tomita, Maggioni, Vogelstein - 2015 - Randomer Forests.pdf:pdf},
pages = {1--9},
title = {{Randomer Forests}},
year = {2015}
}
@article{Andrieu2003,
abstract = {This purpose of this introductory paper is threefold. First, it introduces the Monte Carlo method with emphasis on probabilistic machine learning. Second, it reviews the main building blocks of modern Markov chain Monte Carlo simulation, thereby providing and introduction to the remaining papers of this special issue. Lastly, it discusses new interesting research horizons.},
archivePrefix = {arXiv},
arxivId = {1109.4435v1},
author = {Andrieu, Christophe and {De Freitas}, Nando and Doucet, Arnaud and Jordan, Michael I.},
doi = {10.1023/A:1020281327116},
eprint = {1109.4435v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Andrieu et al. - 2003 - An introduction to MCMC for machine learning.pdf:pdf},
isbn = {08856125 (ISSN)},
issn = {08856125},
journal = {Machine Learning},
keywords = {MCMC,Markov chain Monte Carlo,Sampling,Stochastic algorithms},
number = {1-2},
pages = {5--43},
pmid = {178037200001},
title = {{An introduction to MCMC for machine learning}},
volume = {50},
year = {2003}
}
@article{PedroD,
author = {Domingos, Pedro},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Domingos - 1999 - MetaCost A General Method for Making Classifiers Cost-Sensitive.pdf:pdf},
journal = {Fifth International Conference on Knowledge Discovery and Data Mining},
title = {{MetaCost: A General Method for Making Classifiers Cost-Sensitive}},
year = {1999}
}
@article{Brown2005,
abstract = {Ensemble approaches to classification and regression have attracted a great deal of interest in recent years. These methods can be shown both theoretically and empirically to outperform single predictors on a wide range of tasks. One of the elements required for accurate prediction when using an ensemble is recognised to be error "diversity". However, the exact meaning of this concept is not clear from the literature, particularly for classification tasks. In this paper we first review the varied attempts to provide a formal explanation of error diversity, including several heuristic and qualitative explanations in the literature. For completeness of discussion we include not only the classification literature but also some excerpts of the rather more mature regression literature, which we believe can still provide some insights. We proceed to survey the various techniques used for creating diverse ensembles, and categorise them, forming a preliminary taxonomy of diversity creation methods. As part of this taxonomy we introduce the idea of implicit and explicit diversity creation methods, and three dimensions along which these may be applied. Finally we propose some new directions that may prove fruitful in understanding classification error diversity. © 2004 Elsevier B.V. All rights reserved.},
author = {Brown, Gavin and Wyatt, Jeremy and Harris, Rachel and Yao, Xin},
doi = {10.1016/j.inffus.2004.04.004},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Brown et al. - 2005 - Diversity creation methods A survey and categorisation.pdf:pdf},
issn = {15662535},
journal = {Information Fusion},
keywords = {Diversity,Ensemble,Neural networks,Survey,Taxonomy},
number = {1},
pages = {5--20},
title = {{Diversity creation methods: A survey and categorisation}},
volume = {6},
year = {2005}
}
@article{Ho1998,
abstract = {Much of previous attention on decision trees focuses on the
splitting criteria and optimization of tree sizes. The dilemma between
overfitting and achieving maximum accuracy is seldom resolved. A method
to construct a decision tree based classifier is proposed that maintains
highest accuracy on training data and improves on generalization
accuracy as it grows in complexity. The classifier consists of multiple
trees constructed systematically by pseudorandomly selecting subsets of
components of the feature vector, that is, trees constructed in randomly
chosen subspaces. The subspace method is compared to single-tree
classifiers and other forest construction methods by experiments on
publicly available datasets, where the method's superiority is
demonstrated. We also discuss independence between trees in a forest and
relate that to the combined classification accuracy},
author = {Ho, Tin Kam},
doi = {10.1109/34.709601},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ho - 1998 - The random subspace method for constructing decision forests.pdf:pdf},
isbn = {0162-8828},
issn = {01628828},
journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
keywords = {Bootstrapping,Classifier combination,Decision combination,Decision forest,Decision tree,Multiple-classifier system,Pattern recognition,Stochastic discrimination},
number = {8},
pages = {832--844},
title = {{The random subspace method for constructing decision forests}},
volume = {20},
year = {1998}
}
@article{Casarotto2007,
author = {Casarotto, Chad},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Casarotto - 2007 - Markov Chains and the Ergodic Theorem.pdf:pdf},
pages = {1--7},
title = {{Markov Chains and the Ergodic Theorem}},
year = {2007}
}
@article{Ling2008,
author = {Ling, Charles X. and Sheng, Victor S.},
doi = {10.1.1.15.7095},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ling, Sheng - 2008 - Cost-sensitive learning and the class imbalance problem.pdf:pdf},
isbn = {9780387307688},
journal = {Encyclopedia of Machine Learning},
pages = {231--235},
title = {{Cost-sensitive learning and the class imbalance problem}},
url = {http://www.springer.com/computer/ai/book/978-0-387-30768-8$\backslash$nhttp://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.164.4418\&rep=rep1\&type=pdf},
year = {2008}
}
@article{Utgoff1997,
abstract = {The ability to restructure a decision tree efficiently enables a variety of approaches to decision tree induction that would otherwise be prohibitively expensive. Two such approaches are described here, one being incremental tree induction (ITI), and the other being non-incremental tree induction using a measure of tree quality instead of test quality (DMTI). These approaches and several variants offer new computational and classifier characteristics that lend themselves to particular applications.},
author = {Utgoff, Paul E. and Berkman, Neil C. and Clouse, Jeffery A.},
doi = {10.1.1.53.2413},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Utgoff, Berkman, Clouse - 1997 - Decision Tree Induction Based on Efficient Tree Restructuring.pdf:pdf},
issn = {0885-6125},
journal = {Machine Learning},
keywords = {binary test,decision tree,direct metric,example incorporation,incremental induction,installed test,missing value,tree transposition,update cost,virtual pruning},
number = {C},
pages = {5--44},
title = {{Decision Tree Induction Based on Efficient Tree Restructuring}},
volume = {29},
year = {1997}
}
@article{Schapire1990,
abstract = {The problem of improving the accuracy of a hypothesis output by a
learning algorithm in the distribution-free learning model is
considered. A concept class is learnable (or strongly learnable) if,
given access to a source of examples from the unknown concept, the
learner with high probability is able to output a hypothesis that is
correct on all but an arbitrarily small fraction of the instances. The
concept class is weakly learnable if the learner can produce a
hypothesis that forms only slightly better than random guessing. It is
shown that these two notions of learnability are equivalent. An explicit
method is described for directly converting a weak learning algorithm
into one that achieves arbitrarily high accuracy. This construction may
have practical applications as a tool for efficiently converting a
mediocre learning algorithm into one that performs extremely well. In
addition, the construction has some interesting theoretical consequences
},
author = {Schapire, Robert E.},
doi = {10.1007/BF00116037},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Schapire - 1990 - The strength of weak learnability.pdf:pdf},
isbn = {0-8186-1982-1},
issn = {08856125},
journal = {Machine Learning},
keywords = {Machine learning,PAC learning,learnability theory,learning from examples,polynomial-time identification},
number = {2},
pages = {197--227},
title = {{The strength of weak learnability}},
volume = {5},
year = {1990}
}
@book{Robert2004b,
author = {Robert, Christian P and Casella, George},
booktitle = {Springer, New York. Sanso, B. and Guenni, L},
doi = {10.1007/978-1-4757-4145-2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Robert, Casella - 2004 - Monte Carlo Statistical Methods.djvu:djvu},
isbn = {978-1-4419-1939-7},
keywords = {jabref:noKeywordAssigned},
pages = {1089--1100},
title = {{Monte Carlo Statistical Methods}},
url = {http://books.google.com/books?hl=en\&amp;lr=\&amp;id=HfhGAxn5GugC\&amp;oi=fnd\&amp;pg=PR9\&amp;dq=Monte+Carlo+Statistical+Methods\&amp;ots=Bzt\_0RaSUz\&amp;sig=VCjwbF3WjleVSr63xpMO5oyZXpI$\backslash$nhttp://link.springer.com/10.1007/978-1-4757-4145-2},
volume = {95},
year = {2004}
}
@article{Wang2015,
author = {Wang, Hong and Xu, Qingsong and Zhou, Lifeng},
doi = {10.1371/journal.pone.0117844},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wang, Xu, Zhou - 2015 - Large Unbalanced Credit Scoring Using Lasso-Logistic Regression Ensemble.pdf:pdf},
issn = {1932-6203},
journal = {Plos One},
number = {2},
pages = {e0117844},
title = {{Large Unbalanced Credit Scoring Using Lasso-Logistic Regression Ensemble}},
url = {http://dx.plos.org/10.1371/journal.pone.0117844},
volume = {10},
year = {2015}
}
@article{Lakshminarayanan2014,
abstract = {Ensembles of randomized decision trees, usually referred to as random forests, are widely used for classification and regression tasks in machine learning and statistics. Random forests achieve competitive predictive performance and are computationally efficient to train and test, making them excellent candidates for real-world prediction tasks. The most popular random forest variants (such as Breiman’s random forest and extremely randomized trees) operate on batches of training data. Online methods are now in greater demand. Existing online random forests, however, require more training data than their batch counterpart to achieve comparable predictive performance. In this work, we use Mondrian processes (Roy and Teh, 2009) to construct ensembles of random decision trees we call Mondrian forests. Mondrian forests can be grown in an incremental/online fashion and remarkably, the distribution of online Mondrian forests is the same as that of batch Mondrian forests. Mondrian forests achieve competitive predictive performance comparable with existing online random forests and periodically re-trained batch random forests, while being more than an order of magnitude faster, thus representing a better computation vs accuracy tradeoff},
archivePrefix = {arXiv},
arxivId = {arXiv:1406.2673v1},
author = {Lakshminarayanan, Balaji and Roy, DM and Teh, YW},
eprint = {arXiv:1406.2673v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Lakshminarayanan, Roy, Teh - 2014 - Mondrian Forests Efficient Online Random Forests.pdf:pdf},
journal = {arXiv preprint arXiv:1406.2673},
title = {{Mondrian Forests: Efficient Online Random Forests}},
url = {http://arxiv.org/abs/1406.2673},
year = {2014}
}
@article{Mingers1989,
abstract = {This paper compares five methods for pruning decision trees, developed from sets of examples. When used with uncertain rather than deterministic data, decision-tree induction involves three main stages—creating a complete tree able to classify all the training examples, pruning this tree to give statistical reliability, and processing the pruned tree to improve understandability. This paper concerns the second stage—pruning. It presents empirical comparisons of the five methods across several domains. The results show that three methods—critical value, error complexity and reduced error—perform well, while the other two may cause problems. They also show that there is no significant interaction between the creation and pruning methods.},
author = {Mingers, J.},
doi = {10.1023/A:1022604100933},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Mingers - 1989 - An Empirical Comparison of Pruning Methods for Decision Tree Induction.pdf:pdf},
issn = {0885-6125},
keywords = {HD29 Operational Research - Applications},
pages = {227--243},
title = {{An Empirical Comparison of Pruning Methods for Decision Tree Induction}},
url = {http://dx.doi.org/10.1023/A:1022604100933},
volume = {243},
year = {1989}
}
@article{Barros2011,
abstract = {Decision tree induction algorithms are widely used in knowledge discovery and data mining, specially in scenarios where model comprehensibility is desired. A variation of the traditional univariate approach is the so-called oblique decision tree, which allows multivariate tests in its non-terminal nodes. Oblique decision trees can model decision boundaries that are oblique to the attribute axes, whereas univariate trees can only perform axis-parallel splits. The majority of the oblique and univariate decision tree induction algorithms perform a top-down strategy for growing the tree, relying on an impurity-based measure for splitting nodes. In this paper, we propose a novel bottom-up algorithm for inducing oblique trees named BUTIA. It does not require an impurity-measure for dividing nodes, since we know a priori the data resulting from each split. For generating the splitting hyperplanes, our algorithm implements a support vector machine solution, and a clustering algorithm is used for generating the initial leaves. We compare BUTIA to traditional univariate and oblique decision tree algorithms, C4.5, CART, OC1 and FT, as well as to a standard SVM implementation, using real gene expression benchmark data. Experimental results show the effectiveness of the proposed approach in several cases.},
author = {Barros, Rodrigo C. and Cerri, Ricardo and Jaskowiak, Pablo a. and {De Carvalho}, Andr\'{e} C P L F},
doi = {10.1109/ISDA.2011.6121697},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Barros et al. - 2011 - A bottom-up oblique decision tree induction algorithm.pdf:pdf},
isbn = {9781457716751},
issn = {21647143},
journal = {International Conference on Intelligent Systems Design and Applications, ISDA},
keywords = {SVM,bottom-up induction,clustering,hybrid intelligent systems,oblique decision trees},
pages = {450--456},
title = {{A bottom-up oblique decision tree induction algorithm}},
year = {2011}
}
@article{Joshi2011,
abstract = {We develop an efficient algorithm to implement the adjoint method that computes sensitivities of an interest rate derivative to different underlying rates in the co-terminal swap-rate market model. The order of computation per step of the new method is shown to be proportional to the number of rates times the number of factors, which is the same as the order in the LIBOR market model. ?? 2011 Elsevier B.V.},
author = {Joshi, Mark and Yang, Chao},
doi = {10.1016/j.jedc.2010.12.015},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Joshi, Yang - 2011 - Fast delta computations in the swap-rate market model.pdf:pdf},
issn = {01651889},
journal = {Journal of Economic Dynamics and Control},
keywords = {Adjoint method,Computational order,Delta,Market model,Monte Carlo simulation},
number = {5},
pages = {764--775},
title = {{Fast delta computations in the swap-rate market model}},
volume = {35},
year = {2011}
}
@article{Ishwaran2014,
author = {Ishwaran, Hemant},
doi = {10.1007/s10994-014-5451-2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ishwaran - 2014 - The effect of splitting on random forests.pdf:pdf},
issn = {0885-6125},
journal = {Machine Learning},
number = {1},
pages = {75--118},
title = {{The effect of splitting on random forests}},
url = {http://link.springer.com/10.1007/s10994-014-5451-2},
volume = {99},
year = {2014}
}
@article{Comminges2012,
abstract = {We address the issue of variable selection in the regression model with very high ambient dimension, i.e., when the number of variables is very large. The main focus is on the situation where the number of relevant variables, called intrinsic dimension and denoted by \$d\^{}*\$, is much smaller than the ambient dimension \$d\$. Without assuming any parametric form of the underlying regression function, we get tight conditions making it possible to consistently estimate the set of relevant variables. These conditions relate the intrinsic dimension to the ambient dimension and to the sample size. The procedures that are provably consistent under these tight conditions are simple: they are based on comparing the empirical Fourier coefficients with an appropriately chosen threshold value. The asymptotic analysis reveals the presence of two quite different regimes. The first regime is when \$d\^{}*\$ is fixed. In this case the situation in nonparametric regression is the same as in linear regression, i.e., consistent variable selection is possible if and only if \$\backslash log d\$ is small compared to the sample size \$n\$. The picture is completely different in the second regime, \$d\^{}*\backslash to\backslash infty\$ as \$n\backslash to\backslash infty\$, where we prove that consistent variable selection in nonparametric set-up is possible only if \$d\^{}*+\backslash log\backslash log d\$ is small compared to \$\backslash log n\$.},
archivePrefix = {arXiv},
arxivId = {1106.4293},
author = {Comminges, La\"{e}titia and Dalalyan, Arnak S.},
doi = {10.1214/12-AOS1046},
eprint = {1106.4293},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Comminges, Dalalyan - 2012 - Tight conditions for consistency of variable selection in the context of high dimensionality.pdf:pdf},
issn = {00905364},
journal = {Annals of Statistics},
keywords = {Nonparametric regression,Set estimation,Sparsity pattern,Variable selection},
number = {5},
pages = {2667--2696},
title = {{Tight conditions for consistency of variable selection in the context of high dimensionality}},
volume = {40},
year = {2012}
}
@article{Denil2014,
abstract = {Despite widespread interest and practical use, the theoretical properties of random forests are still not well understood. In this paper we contribute to this understanding in two ways. We present a new theoretically tractable variant of random regression forests and prove that our algorithm is consistent. We also provide an empirical evaluation, comparing our algorithm and other theoretically tractable random forest models to the random forest algorithm used in practice. Our experiments provide insight into the relative importance of different simplifications that theoreticians have made to obtain tractable models for analysis.},
archivePrefix = {arXiv},
arxivId = {arXiv:1310.1415v1},
author = {Denil, Misha and Matheson, David and {De Freitas}, Nando},
eprint = {arXiv:1310.1415v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Denil, Matheson, De Freitas - 2014 - Narrowing the Gap Random Forests In Theory and In Practice.pdf:pdf},
isbn = {9781634393973},
journal = {Proceedings of The 31st International Conference on Machine Learning},
number = {1998},
pages = {665--673},
title = {{Narrowing the Gap: Random Forests In Theory and In Practice}},
url = {http://jmlr.org/proceedings/papers/v32/denil14.html},
year = {2014}
}
@article{Provost2003,
abstract = {Tree induction is one of the most effective and widely used methods for building classification models. However, many applications require cases to be ranked by the probability of class membership. Probability esti- mation trees (PETs) have the same attractive features as classification trees (e.g., comprehensibility, accuracy and efficiency in high dimensions and on large data sets). Unfortunately, decision trees have been found to provide poor probability estimates. Several techniques have been proposed to build more accurate PETs, but, to our knowledge, there has not been a systematic experimental analysis of which techniques actually improve the probability-based rankings, and by how much. In this paper we first discuss why the decision-tree representation is not intrinsically inadequate for probability estimation. Inaccurate probabilities are partially the result of decision-tree induction al- gorithms that focus on maximizing classification accuracy and minimizing tree size (for example via reduced-error pruning). Larger trees can be better for probability estimation, even if the extra size is superfluous for accuracy maximization. We then present the results of a comprehensive set of experiments, testing some straightforward methods for improving probability-based rankings. We show that using a simple, common smoothing method—the Laplace correction—uniformly improves probability-based rankings. In addition, bagging substantially improves the rankings, and is even more effective for this purpose than for improving accuracy. We conclude that PETs, with these simple modifications, should be considered when rankings based on class-membership probability are required.},
author = {Provost, Foster and Domingos, Pedro},
doi = {10.1023/A:1024099825458},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Provost, Domingos - 2003 - Tree induction for probability-based ranking.pdf:pdf},
isbn = {08856125 (ISSN)},
issn = {08856125},
journal = {Machine Learning},
keywords = {Bagging,Classification,Cost-sensitive learning,Decision trees,Laplace correction,Probability estimation,Ranking},
number = {3},
pages = {199--215},
title = {{Tree induction for probability-based ranking}},
volume = {52},
year = {2003}
}
@article{Clemencon2009,
abstract = {This paper investigates how recursive partitioning methods can be adapted to the bipartite ranking problem. In ranking, the pursued goal is global: based on past data, define an order on the whole input space X, so that positive instances take up the top ranks with maximum probability. The most natural way to order all instances consists of projecting the input data onto the real line through a real-valued scoring function s and use the natural order on R. The accuracy of the ordering induced by a candidate <i>s</i> is classically measured in terms of the ROC curve or the AUC. Here we discuss the design of tree-structured scoring functions obtained by recursively maximizing the AUC criterion. The connection with recursive piecewise linear approximation of the optimal ROC curve both in the L<sub>1</sub>-sense and in the L<sub>infin</sub>-sense is highlighted. A novel tree-based algorithm for ranking, called TreeRank, is proposed. Consistency results and generalization bounds of functional nature are established for this ranking method, when considering either the L<sub>1</sub> or L<sub>infin</sub> distance. We also describe committee-based learning procedures using TreeRank as a ldquobase ranker,rdquo in order to overcome obvious drawbacks of such a top-down partitioning technique. Simulation results on artificial data are also displayed.},
author = {Cl\'{e}men\c{c}on, St\'{e}phan and Vayatis, Nicolas},
doi = {10.1109/TIT.2009.2025558},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cl\'{e}men\c{c}on, Vayatis - 2009 - Tree-based ranking methods.pdf:pdf},
issn = {00189448},
journal = {IEEE Transactions on Information Theory},
keywords = {AUC criterion,Adaptive piecewise linear approximation,Bipartite ranking problem,Decision Tree,ROC curve},
number = {9},
pages = {4316--4336},
title = {{Tree-based ranking methods}},
volume = {55},
year = {2009}
}
@article{Bergstra,
author = {Bergstra, James and Bardenet, Remi and Bengio, Yoshua and Kegl, Balazs},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bergstra et al. - Unknown - Algorithms for Hyper-Parameter Optimization.pdf:pdf},
pages = {1--9},
title = {{Algorithms for Hyper-Parameter Optimization}}
}
@article{Kazemi2014,
abstract = {This paper addresses the problem of Face Alignment for a single image. We show how an ensemble of regression trees can be used to estimate the face’s landmark positions directly from a sparse subse ...},
author = {Kazemi, Vahid and Josephine, S},
doi = {10.1109/CVPR.2014.241},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kazemi, Josephine - 2014 - One Millisecond Face Alignment with an Ensemble of Regression Trees.pdf:pdf},
isbn = {978-1-4799-5118-5},
journal = {Computer Vision and Pattern Recognition (CVPR), 2014},
title = {{One Millisecond Face Alignment with an Ensemble of Regression Trees}},
url = {http://www.diva-portal.org/smash/record.jsf?pid=diva2:713097},
year = {2014}
}
@phdthesis{Gao2015,
abstract = {The research area of imbalanced dataset has been attracted increasing attention from both academic and industrial areas, because it poses a serious issues for so many supervised learning problems. Since the number of majority class dominates the number of minority class are from minority class, if training dataset includes all data in order to fit a classic classifier, the classifier tends to classify all data to majority class by ignoring minority data as noise. Thus, it is very significant to select appropriate training dataset in the prepossessing stage for classification of imbalanced dataset. We propose an combination approach of SMOTE (Synthetic Minority Over-sampling Technique) and instance selection approaches. The numeric results show that the proposed combination approach can help classifiers to achieve better performance.},
author = {Gao, Tianxiang},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Gao - 2015 - Hybrid classification approach of SMOTE and instance selection for imbalanced datasets.pdf:pdf},
title = {{Hybrid classification approach of SMOTE and instance selection for imbalanced datasets}},
year = {2015}
}
@article{Capel2009,
author = {Capel, David},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Capel - 2009 - Random Forests and Ferns.pdf:pdf},
title = {{Random Forests and Ferns}},
year = {2009}
}
@article{Steinhaeuser2010b,
abstract = {The analysis of climate data has relied heavily on hypothesis-driven statistical methods, while projections of future climate are based primarily on physics-based computational models. However, in recent years a wealth of new datasets has become available. Therefore, we take a more data-centric approach and propose a unified framework for studying climate, with an aim toward characterizing observed phenomena as well as discovering new knowledge in climate science. Specifically, we posit that complex networks are well suited for both descriptive analysis and predictive modeling tasks. We show that the structural properties of 'climate networks' have useful interpretation within the domain. Further, we extract clusters from these networks and demonstrate their predictive power as climate indices. Our experimental results establish that the network clusters are statistically significantly better predictors than clusters derived using a more traditional clustering approach. Using complex networks as data representation thus enables the unique opportunity for descriptive and predictive modeling to inform each other. 2010 Wiley Periodicals, Inc.},
author = {Steinhaeuser, Karsten and Chawla, Nitesh V and Ganguly, Auroop R},
doi = {10.1002/sam},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Steinhaeuser, Chawla, Ganguly - 2010 - Complex Networks as a Unified Framework for Descriptive Analysis and Predictive Modeling in Clima.pdf:pdf},
issn = {19321872},
journal = {Science And Technology},
keywords = {climate data,community detection,complex networks,multivariate predictive modeling,network analysis},
number = {5},
pages = {497--511},
title = {{Complex Networks as a Unified Framework for Descriptive Analysis and Predictive Modeling in Climate Science}},
volume = {4},
year = {2010}
}
@article{NiteshV.ChawlaArLazarevicLawrenceO.Hall2003,
abstract = {Many real world data mining applications involve learning from imbalanced data sets. Learning from data sets that contain very few instances of the minority (or interesting) class usually produces biased classifiers that have a higher predictive accuracy over the majority class(es), but poorer predictive accuracy over the minority class. SMOTE (Synthetic Minority Over-sampling TEchnique) is specifically designed for learning from imbalanced data sets. This paper presents a novel approach for learning from imbalanced data sets, based on a combination of the SMOTE algorithm and the boosting procedure. Unlike standard boosting where all misclassified examples are given equal weights, SMOTEBoost creates synthetic examples from the rare or minority class, thus indirectly changing the updating weights and compensating for skewed distributions. SMOTEBoost applied to several highly and moderately imbalanced data sets shows improvement in prediction performance on the minority class and overall improved F-values.},
author = {{Nitesh V. Chawla, Ar Lazarevic, Lawrence O. Hall}, Kevin W. Bowyer},
doi = {10.1007/978-3-540-39804-2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Nitesh V. Chawla, Ar Lazarevic, Lawrence O. Hall - 2003 - SMOTEBoost improving prediction of the minority class in boosting.pdf:pdf},
isbn = {3540200851},
issn = {03029743},
journal = {Principles of Knowledge Discovery in Databases, PKDD-2003},
pages = {107--119},
title = {{SMOTEBoost: improving prediction of the minority class in boosting}},
url = {http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.80.1499},
year = {2003}
}
@article{Platt,
author = {Platt, John C.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Platt - 1998 - Sequential Minimal Optimization A Fast Algorithm for Training Support Vector Machines.pdf:pdf},
title = {{Sequential Minimal Optimization: A Fast Algorithm for Training Support Vector Machines}},
url = {http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.43.4376},
year = {1998}
}
@article{Rosenfeld2014,
author = {Rosenfeld, Nir and Meshi, Ofer and Tarlow, Danny and Globerson, Amir},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Rosenfeld et al. - 2014 - Learning Structured Models with the AUC Loss and Its Generalizations.pdf:pdf},
journal = {Aistats},
title = {{Learning Structured Models with the AUC Loss and Its Generalizations}},
volume = {33},
year = {2014}
}
@article{Drish2001a,
abstract = {We use a technique known as binning to convert the outputs of support vector machine (SVM) classifiers into well-calibrated probabilities. Using the KDD98 data set as a testbed, we evaluate predicted probabilities using four metrics, and compare our results to those obtained by Zadrozny and Elkan for naive Bayes classifiers. We stress test the latest release of the LIBSVM software, and use its new functionality for training on unbalanced data sets to find the best parameters for the SVM classifiers. We demonstrate that using the F1 value as a metric for tuning SVMs is successful for making them work on the KDD98 data set,which is highly unbalanced. On the other hand, we show that binning works better for naive Bayes classifiers, and that it is difficult to avoid overfitting directly using SVMs},
author = {Drish, Joseph},
doi = {http://dx.doi.org/10.1.1.161.3828},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Drish - 2001 - Obtaining calibrated probability estimates from support vector machines.pdf:pdf},
journal = {Technical Report University of California at San Diego},
pages = {1--17},
title = {{Obtaining calibrated probability estimates from support vector machines}},
year = {2001}
}
@article{Marcellin,
abstract = {In this paper we present a new entropy measure to grow decision trees. This measure has the characteristic to be asymmetric, allowing the user to grow trees which better correspond to his expectation in terms of recall and precision on each class. Then we propose decision rules adapted to such trees. Experiments have been realized on real medical data from breast cancer screening units},
author = {Marcellin, Simon and Zighed, Djamel A. and Ritschard, Gilbert},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Marcellin, Zighed, Ritschard - 2006 - An asymmetric entropy measure for decision trees.pdf:pdf},
journal = {11th Conference on Information Processing and Management of Uncertainty in Knowledge-Based Systems},
keywords = {ance,asymmetric learning,class imbal-,decision rules,decision trees,entropy},
title = {{An asymmetric entropy measure for decision trees}},
year = {2006}
}
@phdthesis{Louppe2014,
abstract = {Data analysis and machine learning have become an integrative part of the modern scientific methodology, offering automated procedures for the prediction of a phenomenon based on past observations, unraveling underlying patterns in data and providing insights about the problem. Yet, caution should avoid using machine learning as a black-box tool, but rather consider it as a methodology, with a rational thought process that is entirely dependent on the problem under study. In particular, the use of algorithms should ideally require a reasonable understanding of their mechanisms, properties and limitations, in order to better apprehend and interpret their results. Accordingly, the goal of this thesis is to provide an in-depth analysis of random forests, consistently calling into question each and every part of the algorithm, in order to shed new light on its learning capabilities, inner workings and interpretability. The first part of this work studies the induction of decision trees and the construction of ensembles of randomized trees, motivating their design and purpose whenever possible. Our contributions follow with an original complexity analysis of random forests, showing their good computational performance and scalability, along with an in-depth discussion of their implementation details, as contributed within Scikit-Learn. In the second part of this work, we analyse and discuss the interpretability of random forests in the eyes of variable importance measures. The core of our contributions rests in the theoretical characterization of the Mean Decrease of Impurity variable importance measure, from which we prove and derive some of its properties in the case of multiway totally randomized trees and in asymptotic conditions. In consequence of this work, our analysis demonstrates that variable importances [...].},
archivePrefix = {arXiv},
arxivId = {1407.7502},
author = {Louppe, G},
booktitle = {arXiv preprint},
eprint = {1407.7502},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Louppe - 2014 - Understanding random forests from theory to practice.pdf:pdf},
number = {July},
title = {{Understanding random forests from theory to practice}},
url = {http://arxiv.org/abs/1407.7502},
year = {2014}
}
@article{Biaua,
abstract = {This article addresses the problem of functional supervised classifica- tion of Cox process trajectories, whose random intensity is driven by some exogenous random covariable. The classification task is achieved through a regularized convex empirical risk minimization procedure, and a nonasymptotic oracle inequality is derived. We show that the algorithm provides a Bayes-risk consistent classifier. Furthermore, it is proved that the classifier converges at a rate which adapts to the unknown regularity of the intensity process. Our results are obtained by taking advantage of martingale and stochastic calculus arguments, which are natural in this context and fully exploit the functional na- ture of the problem. Index},
archivePrefix = {arXiv},
arxivId = {arXiv:1410.4029v1},
author = {Biau, G\'{e}rard and Cadre, Beno\^{\i}t and Paris, Quentin},
eprint = {arXiv:1410.4029v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Biau, Cadre, Paris - Unknown - Cox Process Functional Learning.pdf:pdf},
keywords = {Cox process,Functional data analysis,consistency,oracle inequality,regularization,stochastic calculus.,supervised classification},
title = {{Cox Process Functional Learning}}
}
@book{Scholkopf2001,
address = {Berlin, Heidelberg},
author = {Scholkopf, Bernhard and Herbrich, Ralf and Smola, Alex J.},
doi = {10.1007/3-540-44581-1},
editor = {Helmbold, David and Williamson, Bob},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Scholkopf, Herbrich, Smola - 2001 - A Generalized Representer Theorem.pdf:pdf},
isbn = {978-3-540-42343-0},
keywords = {Representer theorem},
mendeley-tags = {Representer theorem},
month = sep,
publisher = {Springer Berlin Heidelberg},
series = {Lecture Notes in Computer Science},
title = {{A Generalized Representer Theorem}},
url = {http://www.springerlink.com/index/10.1007/3-540-44581-1},
volume = {2111},
year = {2001}
}
@article{Cieslak2008,
abstract = {Learning from unbalanced datasets presents a convoluted problem in which traditional learning algorithms may perform poorly. The objective functions used for learning the classifiers typically tend to favor the larger, less important classes in such problems. This paper compares the performance of several popular decision tree splitting criteria – information gain, Gini measure, and DKM – and identifies a new skew insensitive measure in Hellinger distance. We outline the strengths of Hellinger distance in class imbalance, proposes its application in forming decision trees, and performs a comprehensive comparative analysis between each decision tree construction method. In addition, we consider the performance of each tree within a powerful sampling wrapper framework to capture the interaction of the splitting metric and sampling. We evaluate over this wide range of datasets and determine which operate best under class imbalance.},
author = {Cieslak, David a. and Chawla, Nitesh V.},
doi = {10.1007/978-3-540-87479-9\_34},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cieslak, Chawla - 2008 - Learning decision trees for unbalanced data.pdf:pdf},
isbn = {354087478X},
issn = {03029743},
journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
number = {PART 1},
pages = {241--256},
title = {{Learning decision trees for unbalanced data}},
volume = {5211 LNAI},
year = {2008}
}
@article{VanderLaan2007a,
abstract = {When trying to learn a model for the prediction of an outcome given a set of covariates, a statistician has many estimation procedures in their toolbox. A few examples of these candidate learners are: least squares, least angle regression, random forests, and spline regression. Previous articles (van der Laan and Dudoit (2003); van der Laan et al. (2006); Sinisi et al. (2007)) theoretically validated the use of cross validation to select an optimal learner among many candidate learners. Motivated by this use of cross validation, we propose a new prediction method for creating a weighted combination of many candidate learners to build the super learner. This article proposes a fast algorithm for constructing a super learner in prediction which uses V-fold cross-validation to select weights to combine an initial set of candidate learners. In addition, this paper contains a practical demonstration of the adaptivity of this so called super learner to various true data generating distributions. This approach for construction of a super learner generalizes to any parameter which can be defined as a minimizer of a loss function.},
author = {van der Laan, Mark J and Polley, Eric C and Hubbard, Alan E},
doi = {10.2202/1544-6115.1309},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/van der Laan, Polley, Hubbard - 2007 - Super Learner.pdf:pdf},
isbn = {1544-6115 (Electronic)$\backslash$n1544-6115 (Linking)},
issn = {1544-6115},
journal = {Statistical applications in genetics and molecular biology},
number = {1},
pages = {Article25},
pmid = {17910531},
title = {{Super Learner}},
volume = {6},
year = {2007}
}
@article{Grimmett-stirzaker,
author = {Grimmett-stirzaker, Reading and Williams, David},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Grimmett-stirzaker, Williams - Unknown - Lecture 9 The Strong Law of Large Numbers.pdf:pdf},
journal = {Most},
number = {Theorem 54},
pages = {47--58},
title = {{Lecture 9 The Strong Law of Large Numbers}}
}
@article{Neapolitan2003,
abstract = {In this first edition book, methods are discussed for doing inference in Bayesian networks and inference diagrams. Hundreds of examples and problems allow readers to grasp the information. Some of the topics discussed include Pearl's message passing algorithm, Parameter Learning: 2 Alternatives, Parameter Learning r Alternatives, Bayesian Structure Learning, and Constraint-Based Learning. For expert systems developers and decision theorists.},
author = {Neapolitan, Richard E},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Neapolitan - 2003 - Learning Bayesian Networks.pdf:pdf},
isbn = {0130125342},
issn = {17485673},
journal = {Molecular Biology},
number = {2},
pages = {674},
title = {{Learning Bayesian Networks}},
url = {http://www.amazon.com/Learning-Bayesian-Networks-Richard-Neapolitan/dp/0130125342},
volume = {6},
year = {2003}
}
@article{Heath1993,
abstract = {This paper introduces a randomized technique for partitioning examples using oblique hyperplanes. Standard decision tree techniques, such as ID3 and its descendants, partition a set of points with axis-parallel hyperplanes. Our method, by contrast, attempts to find hyperplanes at any orientation. The purpose of this more general technique is to find smaller but equally accurate decision trees than those created by other methods. We have tested our algorithm on both real and simulated data, and...},
author = {Heath, David and Kasif, Simon and Salzberg, Steven},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Heath, Kasif, Salzberg - 1993 - Induction of Oblique Decision Trees.pdf:pdf},
isbn = {0262510715},
journal = {Proceedings of the 13th International Joint Conference on Artiffcial Intelligence},
keywords = {decision trees},
number = {410},
pages = {1002--1007},
title = {{Induction of Oblique Decision Trees}},
volume = {21218},
year = {1993}
}
@article{Debarra,
abstract = {Fraud is the use of deception to gain some benefit, often financial gain. Examples of fraud include insurance fraud, credit card fraud, telecommunications fraud, securities fraud, and accounting fraud. Costs for the affected companies are high, and these costs are passed on to their customers. Detection of fraudulent activity is thus critical to control these costs. Last but not least, in order to avoid detection, fraudsters often change their “signatures” (methods of operation). We propose here to address insurance fraud detection via the use of reputation features that characterize insurance claims and ensemble learning to compensate for varying data distributions. We replace each of the original features in the data set with 5 reputation features (RepF): 1) a count of the number of fraudulent claims with the same feature value in the previous 12 months, 2) a count of the number of months in the previous 12 months with a fraudulent claim with the same feature value, 3) a count of the number of legitimate claims with the same feature value in the previous 12 months, 4) a count of the number of months in the previous 12 months with a legitimate claim with the same feature value, and 5) the proportion of claims with the same feature value which are fraudulent in the previous 12 months. Furthermore we use two one-class Support Vector Machines (SVMs) to measure the similarity of the derived reputation feature vector to recently observed fraudulent claims and recently observed legitimate claims. The combined reputation and similarity features are then used to train a Random Forest classifier for new insurance claims. A publicly available auto insurance fraud data set is used to evaluate our approach. Cost savings, the difference in cost for predicting all new insurance claims as non-fraudulent and predicting fraud based on a trained data mining model, are used as our primary evaluation metric. Our approach shows a 13.6\% increase in cost savings compared to previously published state of the art results for the auto insurance fraud data set.},
author = {Debarr, Dave and Wechsler, Harry},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Debarr, Wechsler - Unknown - Fraud Detection Using Reputation Features, SVMs, and Random Forests.pdf:pdf},
keywords = {class support vector machine,cost sensitive,fraud detection,one,random forest,reputation features},
title = {{Fraud Detection Using Reputation Features, SVMs, and Random Forests}}
}
@article{Yang2009,
abstract = {BACKGROUND: Most machine-learning classifiers output label predictions for new instances without indicating how reliable the predictions are. The applicability of these classifiers is limited in critical domains where incorrect predictions have serious consequences, like medical diagnosis. Further, the default assumption of equal misclassification costs is most likely violated in medical diagnosis. RESULTS: In this paper, we present a modified random forest classifier which is incorporated into the conformal predictor scheme. A conformal predictor is a transductive learning scheme, using Kolmogorov complexity to test the randomness of a particular sample with respect to the training sets. Our method show well-calibrated property that the performance can be set prior to classification and the accurate rate is exactly equal to the predefined confidence level. Further, to address the cost sensitive problem, we extend our method to a label-conditional predictor which takes into account different costs for misclassifications in different class and allows different confidence level to be specified for each class. Intensive experiments on benchmark datasets and real world applications show the resultant classifier is well-calibrated and able to control the specific risk of different class. CONCLUSION: The method of using RF outlier measure to design a nonconformity measure benefits the resultant predictor. Further, a label-conditional classifier is developed and turn to be an alternative approach to the cost sensitive learning problem that relies on label-wise predefined confidence level. The target of minimizing the risk of misclassification is achieved by specifying the different confidence level for different class.},
author = {Yang, Fan and Wang, Hua-zhen and Mi, Hong and Lin, Cheng-de and Cai, Wei-wen},
doi = {10.1186/1471-2105-10-S1-S22},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Yang et al. - 2009 - Using random forest for reliable classification and cost-sensitive learning for medical diagnosis.pdf:pdf},
isbn = {1471210510},
issn = {1471-2105},
journal = {BMC bioinformatics},
pages = {S22},
pmid = {19208122},
title = {{Using random forest for reliable classification and cost-sensitive learning for medical diagnosis.}},
volume = {10 Suppl 1},
year = {2009}
}
@article{Robnik-Sikonja2004a,
abstract = {Random forests are one of the most successful ensemble methods which exhibits performance on the level of boosting and support vector machines. The method is fast, robust to noise, does not overfit and offers possibilities for explanation and visualization of its output. We investigate some possibilities to increase strength or decrease correlation of individual trees in the forest. Using several attribute evaluation measures instead of just one gives promising results. On the other hand replacement of ordinary voting with voting weighted with mar- gin achieved on most similar instances gives improvements which are statistically highly significant over several data sets.},
author = {Robnik-Sikonja, Marko},
doi = {10.1007/978-3-540-30115-8\_34},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Robnik-Sikonja - 2004 - Improving random forests.pdf:pdf},
isbn = {978-3-540-23105-9},
issn = {03029743},
journal = {Machine Learning: ECML 2004},
pages = {12},
title = {{Improving random forests}},
url = {http://www.springerlink.com/index/u8bgf0xde638byva.pdf},
year = {2004}
}
@article{Mishina2001,
abstract = {The ability of generalization by random forests is higher than that by other multi-class classifiers because of the effect of bagging and feature selection. Since random forests based on ensemble learning requires a lot of decision trees to obtain high performance, it is not suitablefor implementing the algorithm on the small-scale hardware such as embedded system. In this paper, we propose a boosted random forests in which boosting algorithm is introduced into random forests. Experimental results show that the proposed method, which consists of fewer decision trees, has higher generalizationability comparingto the conventional method.},
author = {Mishina, Yohei and Tsuchiya, Masamitsu and Fujiyoshi, Hironobu},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Mishina, Tsuchiya, Fujiyoshi - 2001 - Boosted Random Forest.pdf:pdf},
keywords = {Boosting,Machine Learning,Pattern Recognition,Random Forest},
number = {3},
pages = {1--2},
title = {{Boosted Random Forest}},
year = {2001}
}
@article{Therneau1997a,
author = {Therneau, Terry M and Atkinson, Elizabeth J},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Therneau, Atkinson - 1997 - An Introduction to Recusive Partitioning Using the RPART Routines.pdf:pdf},
journal = {Mayo Foundation},
pages = {1--62},
title = {{An Introduction to Recusive Partitioning Using the RPART Routines}},
url = {http://www.mayo.edu/hsr/techrpt/61.pdf$\backslash$npapers2://publication/uuid/FBFAD678-1996-4865-9F66-7A3E3921287B},
year = {1997}
}
@article{Mayr2014,
abstract = {The concept of boosting emerged from the field of machine learning. The basic idea is to boost the accuracy of a weak classifying tool by combining various instances into a more accurate prediction. This general concept was later adapted to the field of statistical modelling. This review article attempts to highlight this evolution of boosting algorithms from machine learning to statistical modelling. We describe the AdaBoost algorithm for classification as well as the two most prominent statistical boosting approaches, gradient boosting and likelihood-based boosting. Although both appraoches are typically treated separately in the literature, they share the same methodological roots and follow the same fundamental concepts. Compared to the initial machine learning algorithms, which must be seen as black-box prediction schemes, statistical boosting result in statistical models which offer a straight-forward interpretation. We highlight the methodological background and present the most common software implementations. Worked out examples and corresponding R code can be found in the Appendix.},
archivePrefix = {arXiv},
arxivId = {1403.1452},
author = {Mayr, Andreas and Binder, Harald and Gefeller, Olaf and Schmid, Matthias},
doi = {10.3414/ME13-01-0122},
eprint = {1403.1452},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Mayr et al. - 2014 - The evolution of boosting algorithms - from machine learning to statistical modelling.pdf:pdf},
journal = {arXiv preprint arXiv:1403.1452},
pages = {1--31},
title = {{The evolution of boosting algorithms - from machine learning to statistical modelling}},
url = {http://arxiv.org/abs/1403.1452},
year = {2014}
}
@article{VanHulse2007a,
abstract = {We present a comprehensive suite of experimentation on the subject of learning from imbalanced data. When classes are imbalanced, many learning algorithms can suffer from the perspective of reduced performance. Can data sampling be used to improve the performance of learners built from imbalanced data? Is the effectiveness of sampling related to the type of learner? Do the results change if the objective is to optimize different performance metrics? We address these and other issues in this work, showing that sampling in many cases will improve classifier performance.},
author = {{Van Hulse}, Jason and Khoshgoftaar, Taghi M and Napolitano, Amri},
doi = {10.1145/1273496.1273614},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Van Hulse, Khoshgoftaar, Napolitano - 2007 - Experimental perspectives on learning from imbalanced data.pdf:pdf},
isbn = {9781595937933},
journal = {Proceedings of the 24th International Conference on Machine Learning (2007)},
pages = {935--942},
title = {{Experimental perspectives on learning from imbalanced data}},
url = {http://portal.acm.org/citation.cfm?doid=1273496.1273614},
year = {2007}
}
@article{Marrocco2008a,
abstract = {The majority of the available classification systems focus on the minimization of the classification error rate. This is not always a suitable metric specially when dealing with two-class problems with skewed classes and cost distributions. In this case, an effective criterion to measure the quality of a decision rule is the area under the Receiver Operating Characteristic curve (AUC) that is also useful to measure the ranking quality of a classifier as required in many real applications. In this paper we propose a nonparametric linear classifier based on the maximization of AUC. The approach lies on the analysis of the Wilcoxon-Mann-Whitney statistic of each single feature and on an iterative pairwise coupling of the features for the optimization of the ranking of the combined feature. By the pairwise feature evaluation the proposed procedure is essentially different from other classifiers using AUC as a criterion. Experiments performed on synthetic and real data sets and comparisons with previous approaches confirm the effectiveness of the proposed method. ?? 2007 Elsevier Ltd. All rights reserved.},
author = {Marrocco, C. and Duin, R. P W and Tortorella, F.},
doi = {10.1016/j.patcog.2007.11.017},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Marrocco, Duin, Tortorella - 2008 - Maximizing the area under the ROC curve by pairwise feature combination.pdf:pdf},
isbn = {0031-3203},
issn = {00313203},
journal = {Pattern Recognition},
keywords = {AUC,ROC curve,Ranking,Two-class problems},
number = {6},
pages = {1961--1974},
title = {{Maximizing the area under the ROC curve by pairwise feature combination}},
volume = {41},
year = {2008}
}
@article{Tsymbal2006,
abstract = {Random Forests are a successful ensemble prediction technique that combines two sources of randomness to generate base decision trees; bootstrapping instances for each tree and considering a random subset of features at each node. Breiman in his introductory paper on Random Forests claims that they are more robust than boosting with respect to overfitting noise, and are able to compete with boosting in terms of predictive performance. Multiple recently published empirical studies conducted in various application domains confirm these claims. Random Forests use simple majority voting to combine the predictions of the trees. However, it is clear that each decision tree in a random forest may have different contribution in classifying a certain instance. In this paper, we demonstrate that the prediction performance of Random Forests may still be improved in some domains by replacing the combination function. Dynamic integration, which is based on local performance estimates of base predictors, can be used instead of majority voting. We conduct experiments on a selection of classification datasets, analysing the resulting accuracy, the margin and the bias and variance components of error. The experiments demonstrate that dynamic integration increases accuracy on some datasets. Even if the accuracy remains the same, dynamic integration always increases the margin. A bias/variance decomposition demonstrates that dynamic integration decreases the error by significantly decreasing the bias component while leaving the same or insignificantly increasing the variance. The experiments also demonstrate that the intrinsic similarity measure of Random Forests is better than the commonly used Heterogeneous Euclidean/Overlap Metric in finding a neighbourhood for local estimates in this context.},
author = {Tsymbal, Alexey and Pechenizkiy, Mykola and Cunningham, P\'{a}draig},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Tsymbal, Pechenizkiy, Cunningham - 2006 - Dynamic Integration with Random Forests.pdf:pdf},
isbn = {3-540-45375-X},
issn = {03029743},
journal = {Machine Learning},
pages = {801 -- 808},
title = {{Dynamic Integration with Random Forests}},
year = {2006}
}
@article{Rodriguez2008,
author = {Rodriguez, Yanet and Baets, Bernard De and Garcia, Maria M and Morell, Carlos and Grau, Ricardo},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Rodriguez et al. - 2008 - A Correlation-Based Distance Function for Nearest Neighbor Classification.pdf:pdf},
keywords = {difference metric,distance functions,fuzzy sets theory,nearest neighbour classification,value},
number = {Dvdm},
pages = {284--291},
title = {{A Correlation-Based Distance Function for Nearest Neighbor Classification}},
year = {2008}
}
@article{Sohl-dickstein2015,
abstract = {A central problem in machine learning involves modeling complex data-sets using highly flexi- ble families of probability distributions in which learning, sampling, inference, and evaluation are still analytically or computationally tractable. Here, we develop an approach that simultane- ously achieves both flexibility and tractability. The essential idea, inspired by non-equilibrium statistical physics, is to systematically and slowly destroy structure in a data distribution through an iterative forward diffusion process. We then learn a reverse diffusion process that restores structure in data, yielding a highly flexible and tractable generative model of the data. This ap- proach allows us to rapidly learn, sample from, and evaluate probabilities in deep generative models with thousands of layers or time steps, as well as to compute conditional and posterior probabilities under the learned model. We addi- tionally release an open source reference imple- mentation},
author = {Sohl-dickstein, Jascha and Weiss, Eric A and Maheswaranathan, Niru and Ganguli, Surya},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Sohl-dickstein et al. - 2015 - Deep Unsupervised Learning using Nonequilibrium Thermodynamics.pdf:pdf},
journal = {JMLR},
number = {Proceedings of the 32nd International Conference on Machine Learning},
title = {{Deep Unsupervised Learning using Nonequilibrium Thermodynamics}},
volume = {37},
year = {2015}
}
@article{Fernandez-Delgado2014a,
author = {Fern\'{a}ndez-Delgado, Manuel and Cernadas, Eva and Barro, Sen\'{e}n and Amorim, Dinani},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Fern\'{a}ndez-Delgado et al. - 2014 - Do we Need Hundreds of Classifiers to Solve Real World Classification Problems.pdf:pdf},
journal = {Journal of Machine Learning Research},
pages = {3133--3181},
title = {{Do we Need Hundreds of Classifiers to Solve Real World Classification Problems?}},
volume = {15},
year = {2014}
}
@article{Ishwaran2010,
abstract = {We prove uniform consistency of Random Survival Forests (RSF), a newly introduced forest ensemble learner for analysis of right-censored survival data. Consistency is proven under general splitting rules, bootstrapping, and random selection of variables-that is, under true implementation of the methodology. Under this setting we show that the forest ensemble survival function converges uniformly to the true population survival function. To prove this result we make one key assumption regarding the feature space: we assume that all variables are factors. Doing so ensures that the feature space has finite cardinality and enables us to exploit counting process theory and the uniform consistency of the Kaplan-Meier survival function. ?? 2010.},
archivePrefix = {arXiv},
arxivId = {0811.2844},
author = {Ishwaran, Hemant and Kogalur, Udaya B.},
doi = {10.1016/j.spl.2010.02.020},
eprint = {0811.2844},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ishwaran, Kogalur - 2010 - Consistency of random survival forests.pdf:pdf},
isbn = {0167-7152 (Electronic)$\backslash$r0167-7152 (Linking)},
issn = {01677152},
journal = {Statistics and Probability Letters},
keywords = {Consistency,Ensemble,Factors,Kaplan-Meier,Random forests},
number = {13-14},
pages = {1056--1064},
pmid = {20582150},
title = {{Consistency of random survival forests}},
volume = {80},
year = {2010}
}
@article{Calders2007,
abstract = {In this paper we show an efficient method for inducing classifiers that directly optimize the area under the ROC curve. Recently, AUC gained importance in the classification community as a mean to compare the performance of classifiers. Because most classification methods do not optimize this measure directly, several classification learning methods are emerging that directly optimize the AUC. These methods, however, require many costly computations of the AUC, and hence, do not scale well to large datasets. In this paper, we develop a method to increase the efficiency of computing AUC based on a polynomial approximation of the AUC. As a proof of concept, the approximation is plugged into the construction of a scalable linear classifier that directly optimizes AUC using a gradient descent method. Experiments on real-life datasets show a high accuracy and efficiency of the polynomial approximation. © Springer-Verlag Berlin Heidelberg 2007.},
author = {Calders, Toon and Jaroszewicz, Szymon},
doi = {10.1007/978-3-540-749756-9\_8},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Calders, Jaroszewicz - 2007 - Efficient AUC optimization for classification.pdf:pdf},
isbn = {978-3-540-74975-2},
issn = {0302-9743},
journal = {Knowledge Discovery in Databases},
pages = {42--53},
title = {{Efficient AUC optimization for classification}},
url = {http://link.springer.com/chapter/10.1007/978-3-540-74976-9\_8},
year = {2007}
}
@book{MacKay2005,
abstract = {This book is aimed at senior undergraduates and graduate students in Engineering, Science, Mathematics, and Computing. It expects familiarity with calculus, probability theory, and linear algebra as taught in a rst- or secondyear undergraduate course on mathematics for scientists and engineers. Conventional courses on information theory cover not only the beautiful theoretical ideas of Shannon, but also practical solutions to communication problems. This book goes further, bringing in Bayesian data modelling, Monte Carlo methods, variational methods, clustering algorithms, and neural networks. Why unify information theory and machine learning? Because they are two sides of the same coin. In the 1960s, a single eld, cybernetics, was populated by information theorists, computer scientists, and neuroscientists, all studying common problems. Information theory and machine learning still belong together. Brains are the ultimate compression and communication systems. And the state-of-the-art algorithms for both data compression and error-correcting codes use the same tools as machine learning.},
author = {MacKay, David J C},
booktitle = {Learning},
doi = {10.1198/jasa.2005.s54},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/MacKay - 2005 - Information Theory, Inference, and Learning Algorithms David J.C. MacKay.pdf:pdf},
isbn = {9780521642989},
issn = {01621459},
pages = {1--640},
title = {{Information Theory, Inference, and Learning Algorithms David J.C. MacKay}},
url = {http://pubs.amstat.org/doi/abs/10.1198/jasa.2005.s54$\backslash$nhttp://www.cambridge.org/0521642981},
volume = {100},
year = {2005}
}
@article{Birattari2010,
abstract = {Algorithms for solving hard optimization problems typically have several parameters that need to be set appropriately such that some aspect of performance is optimized. In this chapter, we review F-Race, a racing algorithm for the task of automatic algorithm configuration. F-Race is based on a statistical approach for selecting the best configuration out of a set of candidate configurations under stochastic evaluations. We review the ideas underlying this technique and discuss an extension of the initial F-Race algorithm, which leads to a family of algorithms that we call iterated F-Race. Experimental results comparing one specific implementation of iterated F-Race to the original F-Race algorithm confirm the potential of this family of algorithms.},
author = {Birattari, Mauro and Yuan, Zhi and Balaprakash, Prasanna and St\"{u}tzle, Thomas},
doi = {10.1007/978-3-642-02538-9\_13},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Birattari et al. - 2010 - F-race and iterated F-race An overview.pdf:pdf},
isbn = {9783642025372},
issn = {1781-3794},
journal = {Experimental Methods for the Analysis of Optimization Algorithms},
number = {June},
pages = {311--336},
title = {{F-race and iterated F-race: An overview}},
year = {2010}
}
@article{Qi2012,
abstract = {Modern biology has experienced an increased use of machine learning techniques for large scale and complex biological data analysis. In the area of Bioinformatics, the Random Forest (RF) 6 technique, which includes an ensemble of decision trees and incorporates feature selection and interactions naturally in the learning process, is a popular choice. It is nonparametric, interpretable, efficient, and has high prediction accuracy for many types of data. Recent work in computational biology has seen an increased use of RF, owing to its unique advantages in dealing with small sample size, high-dimensional feature space, and complex data structures.},
author = {Qi, Yanjun},
doi = {10.1007/978-1-4419-9326-7\_11},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Qi - 2012 - Random Forest for Bioinformatics.pdf:pdf},
isbn = {978-1-4419-9325-0},
journal = {Ensemble Machine Learning},
pages = {307--323},
title = {{Random Forest for Bioinformatics}},
year = {2012}
}
@article{Joly2012a,
author = {Joly, Arnaud and Geurts, Pierre and Wehenkel, Louis},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Joly, Geurts, Wehenkel - 2012 - L 1 -based compression of random forest models(2).pdf:pdf;:Users/guillaume/Dropbox/Mendeley Desktop/Joly, Geurts, Wehenkel - 2012 - L 1 -based compression of random forest models.pdf:pdf},
isbn = {9782874190490},
journal = {Esann},
keywords = {2012,e,ensemble of randomized trees,following article,g,high-dimensional supervised learning problems,joly et al,l1-norm regularization,lasso,one page abstract is,pruning,the goal of this,to present the},
number = {April},
pages = {25--27},
title = {{L 1 -based compression of random forest models}},
year = {2012}
}
@article{Dimitriadou2010,
author = {Leisch, Friedrich and Dimitriadou, Evgenia},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Leisch, Dimitriadou - 2010 - Package ‘mlbench’.pdf:pdf},
journal = {Machine Learning Benchmark Problems},
keywords = {Friedrich Leisch \& Evgenia Dimitriadou},
title = {{Package ‘mlbench’}},
url = {http://cran.r-project.org/web/packages/mlbench},
year = {2010}
}
@misc{Geman1992,
abstract = {Feedforward neural networks trained by error backpropagation are examples of nonparametric regression estimators. We present a tutorial on nonparametric inference and its relation to neural networks, and we use the statistical viewpoint to highlight strengths and weaknesses of neural models. We illustrate the main points with some recognition experiments involving artificial data as well as handwritten numerals. In way of conclusion, we suggest that current-generation feedforward neural networks are largely inadequate for difficult problems in machine perception and machine learning, regardless of parallel-versus-serial hardware or other implementation issues. Furthermore, we suggest that the fundamental challenges in neural modeling are about representation rather than learning per se. This last point is supported by additional experiments with handwritten numerals.},
author = {Geman, Stuart and Bienenstock, Elie and Doursat, Ren\'{e}},
booktitle = {Neural Computation},
doi = {10.1162/neco.1992.4.1.1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Geman, Bienenstock, Doursat - 1992 - Neural Networks and the BiasVariance Dilemma.pdf:pdf},
isbn = {0899-7667},
issn = {0899-7667},
number = {1},
pages = {1--58},
title = {{Neural Networks and the Bias/Variance Dilemma}},
volume = {4},
year = {1992}
}
@techreport{Bellkor2008,
author = {Bell, Robert M. and Koren, Yehuda and Volinsky, Chris},
doi = {10.1111/j.1754-4505.2009.00107.x},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bell, Koren, Volinsky - 2008 - The BellKor 2008 Solution to the Netflix Prize.pdf:pdf},
issn = {0275-1879},
pmid = {19886931},
title = {{The BellKor 2008 Solution to the Netflix Prize}},
year = {2008}
}
@article{Giles2006,
abstract = {Monte Carlo calculation of price sensitivities for hedging is often very time-consuming. Michael Giles and Paul Glasserman develop an adjoint method to accelerate the calculation. The method is particularly effective in estimating sensitivities to a large number of inputs, such as initial rates on a forward curve or points on a volatility surface. The authors apply the method to the Libor market model and show that it is much faster than previous methods.},
author = {Giles, M and Glasserman, P},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Giles, Glasserman - 2006 - Smoking adjoints fast monte carlo greeks.pdf:pdf},
journal = {Risk},
number = {3},
pages = {88--92},
title = {{Smoking adjoints: fast monte carlo greeks}},
url = {http://scholar.google.com/scholar?hl=en\&btnG=Search\&q=intitle:Smoking+adjoints:+fast+Monte+Carlo+Greeks\#0},
year = {2006}
}
@article{Beygelzimer2015,
author = {Beygelzimer, Alina and Kale, Satyen and Luo, Haipeng},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Beygelzimer, Kale, Luo - 2015 - Optimal and Adaptive Algorithms for Online Boosting.pdf:pdf},
journal = {International Conference on Machine Learning},
title = {{Optimal and Adaptive Algorithms for Online Boosting}},
volume = {37},
year = {2015}
}
@techreport{Bernstein,
abstract = {We introduce new methods for optimizing the performance of machine learning classifiers. Building from the property that prediction algorithms interpret a model output by a learner, we explore optimizations that can be made during prediction by replacing the interpreter with a compiler. We explore this idea in the context of decision-tree classifiers and use our findings to discuss potential optimizations that could be made in the context of other models. We validate our optimizations on predictors that are used by several classes of applications, including a decision tree classifier used in Prefab, a research project lead by one of the authors.},
author = {Bernstein, Gilbert and Dixon, Morgan and Levy, Amit},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bernstein, Dixon, Levy - Unknown - Faster Real-Time Classification Using Compilation Final Report Compile Time Run Time.pdf:pdf},
title = {{Faster Real-Time Classification Using Compilation [ Final Report ] Compile Time Run Time}}
}
@article{Adnan2014,
abstract = {Random Forest is one of the most popular decision forest building algorithms that use decision trees as the base classifiers. The splitting attributes for decision trees of Random Forest are generally determined from a predefined number of randomly selected attribute subset of the original attribute set. In this paper, we propose a new technique that randomly determines the size of the at- tribute subset between a dynamically determined range based on the relative size of current data segment to the bootstrap samples at each node splitting event. We present elaborate experimental results involving five widely used da- ta sets from the UCI Machine Learning Repository. The experimental results indicate the effectiveness of the proposed technique in the context of Random Forest. Keywords:},
author = {Adnan, Nasim},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Adnan - 2014 - On Dynamic Selection of Subspace for Random Forest.pdf:pdf},
keywords = {decision forest,decision tree,prediction accuracy,random forest},
pages = {370--379},
title = {{On Dynamic Selection of Subspace for Random Forest}},
year = {2014}
}
@techreport{Kondor2008,
author = {Kondor, Risi},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kondor - 2008 - Vapnik-Chervonenkis theory.pdf:pdf},
number = {3},
pages = {1--6},
title = {{Vapnik-Chervonenkis theory}},
year = {2008}
}
@article{Wei2013,
abstract = {Training and testing of conventional machine learning models on binary classification problems depend on the proportions of the two outcomes in the relevant data sets. This may be especially important in practical terms when real-world applications of the classifier are either highly imbalanced or occur in unknown proportions. Intuitively, it may seem sensible to train machine learning models on data similar to the target data in terms of proportions of the two binary outcomes. However, we show that this is not the case using the example of prediction of deleterious and neutral phenotypes of human missense mutations in human genome data, for which the proportion of the binary outcome is unknown. Our results indicate that using balanced training data (50\% neutral and 50\% deleterious) results in the highest balanced accuracy (the average of True Positive Rate and True Negative Rate), Matthews correlation coefficient, and area under ROC curves, no matter what the proportions of the two phenotypes are in the testing data. Besides balancing the data by undersampling the majority class, other techniques in machine learning include oversampling the minority class, interpolating minority-class data points and various penalties for misclassifying the minority class. However, these techniques are not commonly used in either the missense phenotype prediction problem or in the prediction of disordered residues in proteins, where the imbalance problem is substantial. The appropriate approach depends on the amount of available data and the specific problem at hand.},
author = {Wei, Qiong and Dunbrack, Roland L.},
doi = {10.1371/journal.pone.0067863},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Wei, Dunbrack - 2013 - The Role of Balanced Training and Testing Data Sets for Binary Classifiers in Bioinformatics.pdf:pdf},
issn = {19326203},
journal = {PLoS ONE},
number = {7},
pmid = {23874456},
title = {{The Role of Balanced Training and Testing Data Sets for Binary Classifiers in Bioinformatics}},
volume = {8},
year = {2013}
}
@article{Dzeroski2004,
abstract = {Weempirically evaluate several state-of-the-art methods for constructing ensembles of heterogeneous classifiers with stacking and show that they perform (at best) comparably to selecting the best classifier from the ensemble by cross validation.Amongstate-of-the-art stacking methods, stacking with probability distributions and multi-response linear regression performs best.We propose two extensions of this method, one using an extended set of meta-level features and the other using multi-response model trees to learn at the meta-level.We show that the latter extension performs better than existing stacking approaches and better than selecting the best classifier by cross validation. Keywords:},
author = {Dzeroski, Saso and Zenko, Bernard},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Dzeroski, Zenko - 2004 - Is Combining Classifiers with Stacking Better than Selecting the Best One.pdf:pdf},
keywords = {combining classifiers,ensembles of classifiers,meta-learning,multi-response model trees,stacking},
pages = {255--273},
title = {{Is Combining Classifiers with Stacking Better than Selecting the Best One ?}},
year = {2004}
}
@article{Knuth1992,
abstract = {The author advocates two specific mathematical notations from his popular course and joint textbook, "Concrete Mathematics". The first of these, extending an idea of Iverson, is the notation "[P]" for the function which is 1 when the Boolean condition P is true and 0 otherwise. This notation can encourage and clarify the use of characteristic functions and Kronecker deltas in sums and integrals.   The second notation puts Stirling numbers on the same footing as binomial coefficients. Since binomial coefficients are written on two lines in parentheses and read "n choose k", Stirling numbers of the first kind should be written on two lines in brackets and read "n cycle k", while Stirling numbers of the second kind should be written in braces and read "n subset k". (I might say "n partition k".) The written form was first suggested by Imanuel Marx. The virtues of this notation are that Stirling partition numbers frequently appear in combinatorics, and that it more clearly presents functional relations similar to those satisfied by binomial coefficients.},
archivePrefix = {arXiv},
arxivId = {math/9205211},
author = {Knuth, Donald E.},
doi = {10.2307/2325085},
eprint = {9205211},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Knuth - 1992 - Two notes on notation.pdf:pdf},
issn = {00029890},
pages = {1--23},
primaryClass = {math},
title = {{Two notes on notation}},
url = {http://arxiv.org/abs/math/9205211},
year = {1992}
}
@article{Dietterich1998,
abstract = {This article reviews ve approximate statistical tests for determining whether one learning algorithm outperforms another on a particular learn- ing task. These tests are compared experimentally to determine their prob- ability of incorrectly detecting a difference when no difference exists (type I error). Two widely used statistical tests are shown to have high probability of type I error in certain situations and should never be used: a test for the difference of two proportions and a paired-differences t test based on taking several random train-test splits. A third test, a paired- differences t test based on 10-fold cross-validation, exhibits somewhat elevated probability of type I error. A fourth test, McNemar’s test, is shown to have low type I error. The fth test is a new test, 5 £ 2 cv, based on ve iterations of twofold cross-validation. Experiments show that this test also has acceptable type I error. The article also measures the power (ability to detect algorithm differences when they do exist) of these tests. The cross-validated t test is the most powerful. The 5 £2 cv test is shown to be slightly more powerful than McNemar’s test. The choice of the best test is determined by the computational cost of running the learning algorithm. For algorithms that can be executed only once, Mc- Nemar’s test is the only test with acceptable type I error. For algorithms that can be executed 10 times, the 5 £ 2 cv test is recommended, because it is slightly more powerful and because it directly measures variation due to the choice of training set.},
author = {Dietterich, Thomas G},
doi = {10.1162/089976698300017197},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Dietterich - 1998 - Approximate Statistical Tests for Comparing Supervised Classication Learning Algorithms.pdf:pdf},
issn = {0899-7667},
journal = {Neural Computation},
pages = {1895--1923},
title = {{Approximate Statistical Tests for Comparing Supervised Classication Learning Algorithms}},
volume = {10},
year = {1998}
}
@article{Vijayakumar1998,
author = {Vijayakumar, Sethu and Schaal, Stefan and Science, Computer and Angeles, Los},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Vijayakumar et al. - 1998 - Bayesian Backfitting.pdf:pdf},
number = {1},
title = {{Bayesian Backfitting}},
year = {1998}
}
@article{Kursa2014b,
abstract = {Random ferns is a very simple yet powerful classification method orig-inally introduced for specific computer vision tasks. In this paper, I show that this algorithm may be considered as a constrained decision tree en-semble and use this interpretation to introduce a series of modifications allowing one to use Random ferns in a general machine learning prob-lems. Moreover, I extend the method with internal error approximation and attribute importance measure based on a corresponding features of the Random forest algorithm. I also present the R package rFerns containing an efficient implemen-tation of such modified version of Random ferns.},
archivePrefix = {arXiv},
arxivId = {arXiv:1202.1121v2},
author = {Kursa, Miron Bartosz},
eprint = {arXiv:1202.1121v2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kursa - 2014 - rFerns – Random Ferns Method Implementation for the General-Purpose Machine Learning.pdf:pdf},
pages = {1--13},
title = {{rFerns – Random Ferns Method Implementation for the General-Purpose Machine Learning}},
year = {2014}
}
@article{Biau2010a,
archivePrefix = {arXiv},
arxivId = {arXiv:1005.0208v3},
author = {Biau, G\'{e}rard},
eprint = {arXiv:1005.0208v3},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Biau - 2010 - Analysis of a Random Forests Model.pdf:pdf},
journal = {Mathematics Subject Classification},
pages = {1--40},
title = {{Analysis of a Random Forests Model}},
year = {2010}
}
@book{Bishop2006,
archivePrefix = {arXiv},
arxivId = {0-387-31073-8},
author = {Bishop, Christopher M.},
booktitle = {Springer},
doi = {10.1641/B580519},
eprint = {0-387-31073-8},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Bishop - 2006 - Pattern Recognition and Machine Learning.pdf:pdf},
isbn = {9780387310732},
issn = {00063568},
pmid = {18292226},
title = {{Pattern Recognition and Machine Learning}},
url = {http://www.library.wisc.edu/selectedtocs/bg0137.pdf},
year = {2006}
}
@article{Batt2011,
author = {Rosenthal, Jeffrey S.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Rosenthal - 2011 - MCMC Theory What Is It Good For.pdf:pdf},
pages = {22--25},
title = {{MCMC Theory: What Is It Good For ?}},
year = {2011}
}
@article{Sill2009,
abstract = {Ensemble methods, such as stacking, are designed to boost predictive accuracy by blending the predictions of multiple machine learning models. Recent work has shown that the use of meta-features, additional inputs describing each example in a dataset, can boost the performance of ensemble methods, but the greatest reported gains have come from nonlinear procedures requiring significant tuning and training time. Here, we present a linear technique, Feature-Weighted Linear Stacking (FWLS), that incorporates meta-features for improved accuracy while retaining the well-known virtues of linear regression regarding speed, stability, and interpretability. FWLS combines model predictions linearly using coefficients that are themselves linear functions of meta-features. This technique was a key facet of the solution of the second place team in the recently concluded Netflix Prize competition. Significant increases in accuracy over standard linear stacking are demonstrated on the Netflix Prize collaborative filtering dataset.},
archivePrefix = {arXiv},
arxivId = {0911.0460},
author = {Sill, Joseph and Takacs, Gabor and Mackey, Lester and Lin, David},
eprint = {0911.0460},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Sill et al. - 2009 - Feature-Weighted Linear Stacking.pdf:pdf},
pages = {17},
title = {{Feature-Weighted Linear Stacking}},
url = {http://arxiv.org/abs/0911.0460},
year = {2009}
}
@article{Robnik-Sikonja2004,
abstract = {Random forests are one of the most successful ensemble methods which exhibits performance on the level of boosting and support vector machines. The method is fast, robust to noise, does not overfit and offers possibilities for explanation and visualization of its output. We investigate some possibilities to increase strength or decrease correlation of individual trees in the forest. Using several attribute evaluation measures instead of just one gives promising results. On the other hand replacement of ordinary voting with voting weighted with mar- gin achieved on most similar instances gives improvements which are statistically highly significant over several data sets.},
author = {Robnik-Sikonja, Marko},
doi = {10.1007/978-3-540-30115-8\_34},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Robnik-Sikonja - 2004 - Improving random forests(2).pdf:pdf},
isbn = {978-3-540-23105-9},
issn = {03029743},
journal = {Machine Learning: ECML 2004},
pages = {12},
title = {{Improving random forests}},
url = {http://www.springerlink.com/index/u8bgf0xde638byva.pdf},
year = {2004}
}
@misc{Ishwaran2015,
author = {Ishwaran, Hemant and Kogalur, Udaya B.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ishwaran, Kogalur - 2015 - Package ‘ randomForestSRC ’.pdf:pdf},
title = {{Package ‘ randomForestSRC ’}},
year = {2015}
}
@article{Elliott2009,
abstract = {Automatic differentiation (AD) is a precise, efficient, and convenient method for computing derivatives of functions. Its forward-mode implementation can be quite simple even when extended to compute all of the higher-order derivatives as well. The higher-dimensional case has also been tackled, though with extra complexity. This paper develops an implementation of higher-dimensional, higher-order, forward-mode AD in the extremely general and elegant setting of calculus on manifolds and derives that implementation from a simple and precise specification. In order to motivate and discover the implementation, the paper poses the question "What does AD mean, independently of implementation?" An answer arises in the form of naturality of sampling a function and its derivative. Automatic differentiation flows out of this naturality condition, together with the chain rule. Graduating from first-order to higher-order AD corresponds to sampling all derivatives instead of just one. Next, the setting is expanded to arbitrary vector spaces, in which derivative values are linear maps. The specification of AD adapts to this elegant and very general setting, which even simplifies the development.},
author = {Elliott, Conal M.},
doi = {10.1145/1631687.1596579},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Elliott - 2009 - Beautiful differentiation.pdf:pdf},
isbn = {9781605583327},
issn = {03621340},
journal = {ACM SIGPLAN Notices},
keywords = {automatic differentiation,program derivation},
number = {9},
pages = {191},
title = {{Beautiful differentiation}},
volume = {44},
year = {2009}
}
@article{Biau2008,
abstract = {In the last years of his life, Leo Breiman promoted random forests for use in classification. He 
suggested using averaging as a means of obtaining good discrimination rules. The base classifiers 
used for averaging are simple and randomized, often based on random samples from the data. He 
left a few questions unanswered regarding the consistency of such rules. In this paper, we give a 
number of theorems that establish the universal consistency of averaging rules. We also show that 
some popular classifiers, including one suggested by Breiman, are not universally consistent.},
author = {Biau, G\'{e}rard and Devroye, Luc and Lugosi, G\'{a}bor},
doi = {10.1145/1390681.1442799},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Biau, Devroye, Lugosi - 2008 - Consistency of random forests and other averaging classifiers.pdf:pdf},
isbn = {1532-4435},
issn = {1532-4435},
keywords = {Computational, Information-Theoretic Learning with,Learning/Statistics \& Optimisation,Theory \& Algorithms},
number = {506778},
title = {{Consistency of random forests and other averaging classifiers}},
url = {http://eprints.pascal-network.org/archive/00004623/},
year = {2008}
}
@article{Ehrlinger2014,
abstract = {Random Forests (Breiman 2001) (RF) are a non-parametric statistical method requir- ing no distributional assumptions on covariate relation to the response. RF are a robust, nonlinear technique that optimizes predictive accuracy by fitting an ensemble of trees to stabilize model estimates. The randomForestSRC package (Ishwaran and Kogalur 2014) is a unified treatment of Breiman’s random forests for survival, regression and classification problems. Predictive accuracy make RF an attractive alternative to parametric models, though complexity and interpretability of the forest hinder wider application of the method. We introduce the ggRandomForests package, tools for visually understand random for- est models grown in R (R Core Team 2014) with the randomForestSRC package. The ggRandomForests package is structured to extract intermediate data objects from ran- domForestSRC objects and generates figures using the ggplot2 (Wickham 2009) graphics package. This document is structured as a tutorial for building random forests for regression with the randomForestSRC package and using the ggRandomForests package for inves- tigating how the forest is constructed. We investigate the Boston Housing data (Har- rison and Rubinfeld 1978; Belsley, Kuh, and Welsch 1980). We demonstrate random forest variable selection using Variable Importance (VIMP) (Breiman 2001) and Mini- mal Depth (Ishwaran, Kogalur, Gorodeski, Minn, and Lauer 2010), a property derived from the construction of each tree within the forest. We will also demonstrate the use of variable dependence plots (Friedman 2000) to aid interpretation RF results. We then examine variable interactions between covariates using a minimal depth interactions, and conditional variable dependence plots. The goal of the exercise is to demonstrate the strength of using Random Forest methods for both prediction and information retrieval in regression settings. Keywords:},
archivePrefix = {arXiv},
arxivId = {arXiv:1501.07196v2},
author = {Ehrlinger, John},
eprint = {arXiv:1501.07196v2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ehrlinger - 2014 - ggRandomForests Random Forests for Regression.pdf:pdf},
keywords = {minimal depth,r,random forest,randomforestsrc,regression,vimp},
number = {Breiman 2001},
title = {{ggRandomForests : Random Forests for Regression}},
year = {2014}
}
@article{Cieslak2006,
author = {Cieslak, D.a. and Chawla, N.V.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cieslak, Chawla - 2006 - Calibration and Power of Probability Estimation Trees on Unbalanced Data.pdf:pdf},
journal = {Cse.Nd.Edu},
title = {{Calibration and Power of Probability Estimation Trees on Unbalanced Data}},
url = {http://www.cse.nd.edu/Reports/2006/TR-2006-12.pdf},
year = {2006}
}
@article{Cover1965,
abstract = {This paper develops the separating capacities of families of nonlinear decision surfaces by a direct application of a theorem in classical combinatorial geometry. It is shown that a family of surfaces having d degrees of freedom has a natural separating capacity of 2d pattern vectors, thus extending and unifying results of Winder and others on the pattern-separating capacity of hyperplanes. Applying these ideas to the vertices of a binary n-cube yields bounds on the number of spherically, quadratically, and, in general, nonlinearly separable Boolean functions of n variables. It is shown that the set of all surfaces which separate a dichotomy of an infinite, random, separable set of pattern vectors can be characterized, on the average, by a subset of only 2d extreme pattern vectors. In addition, the problem of generalizing the classifications on a labeled set of pattern points to the classification of a new point is defined, and it is found that the probability of ambiguous generalization is large unless the number of training patterns exceeds the capacity of the set of separating surfaces.},
author = {Cover, Thomas M.},
doi = {10.1109/PGEC.1965.264137},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cover - 1965 - Geometrical and Statistical Properties of Systems of Linear Inequalities with Applications in Pattern Recognition.pdf:pdf},
issn = {0367-7508},
journal = {IEEE Transactions on Electronic Computers},
keywords = {Application software,Boolean functions,Geometry,History,Pattern recognition,Vectors},
number = {3},
pages = {326--334},
shorttitle = {Electronic Computers, IEEE Transactions on},
title = {{Geometrical and Statistical Properties of Systems of Linear Inequalities with Applications in Pattern Recognition}},
url = {http://ieeexplore.ieee.org/lpdocs/epic03/wrapper.htm?arnumber=4038449},
volume = {EC-14},
year = {1965}
}
@techreport{Man2014,
author = {Man, Ramon},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Man - 2014 - Survival analysis in credit scoring A framework for PD estimation.pdf:pdf},
institution = {University of Twente},
title = {{Survival analysis in credit scoring A framework for PD estimation}},
year = {2014}
}
@article{Kuncheva2009,
abstract = {Rotation Forest is a recently proposed method for building classifier ensembles using independently trained decision trees. It was found to be more accurate than bagging, AdaBoost and Random Forest ensembles across a collection of benchmark data sets. This paper carries out a lesion study on Rotation Forest in order to find out which of the parameters and the randomization heuristics are responsible for the good performance. Contrary to common intuition, the features extracted through PCA gave the best results compared to those extracted through non-parametric discriminant analysis (NDA) or random projections. The only ensemble method whose accuracy was statistically indistinguishable from that of Rotation Forest was LogitBoost although it gave slightly inferior results on 20 out of the 32 benchmark data sets. It appeared that the main factor for the success of Rotation Forest is that the transformation matrix employed to calculate the (linear) extracted features is sparse.},
author = {Kuncheva, L.I. and Rodriguez, J.J.},
doi = {10.1007/978-3-540-72523-7},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kuncheva, Rodriguez - 2009 - An experimental study on Rotation Forest ensembles.pdf:pdf},
isbn = {978-3-540-72481-0},
issn = {03029743},
keywords = {Computer},
pages = {459--468},
title = {{An experimental study on Rotation Forest ensembles.}},
url = {http://hdl.handle.net/10242/41916},
year = {2009}
}
@article{Schapire2009,
abstract = {Boosting is a general method for improving the accuracy of any given learning algorithm. This short overview paper introduces the boosting algorithm AdaBoost, and explains the underlying theory of boosting, including an explanation of why boosting often does not suffer from overfitting as well as boostings relationship to support-vector machines. Some examples of recent applications of boosting are also described.},
archivePrefix = {arXiv},
arxivId = {arXiv:1508.01136v1},
author = {Schapire, Robert E and Freund, Yoav},
eprint = {arXiv:1508.01136v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Schapire, Freund - 2009 - A Short Introduction to Boosting.pdf:pdf},
isbn = {9090110658},
number = {5},
pages = {771--780},
title = {{A Short Introduction to Boosting}},
volume = {14},
year = {2009}
}
@article{Bucilua2006,
abstract = {Often the best performing supervised learning models are ensembles of hundreds or thousands of base-level classifiers. Unfortunately, the space required to store this many classifiers, and the time required to execute them at run-time, prohibits their use in applications where test sets are large (e.g. Google), where storage space is at a premium (e.g. PDAs), and where computational power is limited (e.g. hea-ring aids). We present a method for "compressing" large, complex ensembles into smaller, faster models, usually without significant loss in performance.},
author = {Buciluǎ, Cristian and Caruana, Rich and Niculescu-Mizil, Alexandru},
doi = {10.1145/1150402.1150464},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Buciluǎ, Caruana, Niculescu-Mizil - 2006 - Model compression.pdf:pdf},
isbn = {1595933395},
journal = {Proceedings of the 12th ACM SIGKDD international conference on Knowledge discovery and data mining - KDD '06},
keywords = {model compression,supervised learning},
pages = {535},
title = {{Model compression}},
url = {http://portal.acm.org/citation.cfm?doid=1150402.1150464},
year = {2006}
}
@article{Kotlowski2011,
abstract = {Minimization of the rank loss or, equivalently, maximization of the AUC in bipartite ranking calls for minimizing the number of disagreements between pairs of instances. Since the complexity of this problem is inherently
quadratic in the number of training examples, it is tempting to ask how much is actually lost by minimizing a simple univariate loss function, as done by standard classification methods, as a surrogate. In this paper, we first note that minimization of 0/1 loss is not an option, as it may yield an arbitrarily high rank loss. We show, however, that better results can be achieved by means of a weighted (cost-sensitive) version of 0/1 loss. Yet, the real gain is obtained through marginbased loss functions, for which we are able to derive proper bounds, not only for rank risk but, more importantly, also for rank regret. The paper is completed with an experimental study in which we address specific questions raised by our theoretical analysis.},
author = {Kotlowski, Wojciech and Dembczynski, Krzysztof and Huellermeier, Eyke},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kotlowski, Dembczynski, Huellermeier - 2011 - Bipartite Ranking through Minimization of Univariate Loss.pdf:pdf},
isbn = {978-1-4503-0619-5},
keywords = {Learning/Statistics \& Optimisation},
title = {{Bipartite Ranking through Minimization of Univariate Loss}},
url = {http://eprints.pascal-network.org/archive/00009277/},
year = {2011}
}
@article{Liang2007,
author = {Liang, Guohua and Cohn, Anthony G},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Liang, Cohn - 2007 - An Effective Approach for Imbalanced Classification Unevenly Balanced Bagging.pdf:pdf},
isbn = {1304201201},
journal = {Proceedings of the Twenty-Seventh AAAI Conference on Artificial Intelligence An},
keywords = {Student Abstract and Poster Program},
pages = {1633--1634},
title = {{An Effective Approach for Imbalanced Classification : Unevenly Balanced Bagging}},
year = {2007}
}
@article{Minka2002,
author = {Minka, Thomas P.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Minka - 2002 - Bayesian model averaging is not model combination.pdf:pdf},
number = {2000},
pages = {1--2},
title = {{Bayesian model averaging is not model combination}},
year = {2002}
}
@article{Fast2008,
abstract = {Collective classification techniques jointly infer all class labels of a relational data set, using the inferences about one class label to influence inferences about related class labels. Kou and Cohen recently introduced an efficient relational model based on stacking that, despite its simplicity, has equivalent accuracy to more sophisticated joint inference approaches. Using experiments on both real and synthetic data, we show that the primary cause for the performance of the stacked model is the reduction in bias from learning the stacked model on inferred labels rather than true labels. The reduction in variance due to conditional inference also contributes to the effect but it is not as strong. In addition, we show that the performance of the joint inference and stacked learners can be attributed to an implicit weighting of local and relational features at learning time.},
author = {Fast, Andrew and Jensen, David},
doi = {10.1109/ICDM.2008.126},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Fast, Jensen - 2008 - Why stacked models perform effective collective classification.pdf:pdf},
isbn = {9780769535029},
issn = {15504786},
journal = {Proceedings - IEEE International Conference on Data Mining, ICDM},
pages = {785--790},
title = {{Why stacked models perform effective collective classification}},
year = {2008}
}
@phdthesis{Thomas2009,
author = {Thomas, Julien},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Thomas - 2009 - Apprentissage supervis\'{e} de donn\'{e}es d\'{e}s\'{e}quilibr\'{e}es par for\^{e}t al\'{e}atoire .pdf:pdf},
title = {{Apprentissage supervis\'{e} de donn\'{e}es d\'{e}s\'{e}quilibr\'{e}es par for\^{e}t al\'{e}atoire .}},
year = {2009}
}
@article{Efron2003,
author = {Efron, Bradley and Hastie, Trevor and Johnstone, Iain and Tibshirani, Robert},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Efron et al. - 2003 - Least Angle Regression.pdf:pdf},
journal = {Annals of Statistics},
pages = {1--44},
title = {{Least Angle Regression}},
year = {2003}
}
@article{Zhou2012a,
abstract = {In this paper, we propose an improved random forest algorithm which allocates weights to decision trees in the forest during tree aggregation for prediction and their weights are easily calculated based on out-of-bag errors in training. Experiments results show that our proposed algorithm beats the original random forest and other popular classification algorithms such as SVM, KNN and C4.5 in terms of both balanced and overall accuracy metrics. Experiments also show that parallel random forests can greatly improve random forests’ efficiency during the learning process.},
author = {Zhou, Lifeng and Wang, Hong},
doi = {10.11591/telkomnika.v10i6.1323},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Zhou, Wang - 2012 - Loan Default Prediction on Large Imbalanced Data Using Random Forests.pdf:pdf;:Users/guillaume/Dropbox/Mendeley Desktop/Zhou, Wang - 2012 - Loan Default Prediction on Large Imbalanced Data Using Random Forests(2).pdf:pdf},
issn = {2087-278X},
keywords = {2012 universitas ahmad dahlan,all rights reserved,copyright,imbalanced data,loan default prediction,parallel computing,random forests},
number = {6},
pages = {1519--1525},
title = {{Loan Default Prediction on Large Imbalanced Data Using Random Forests}},
volume = {10},
year = {2012}
}
@article{Nguyen2005,
author = {Nguyen, Canh Hao and Nguyen, Canh Hao and Ho, Tu Bao and Ho, Tu Bao},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Nguyen et al. - 2005 - An Imbalanced Data Rule Learner.pdf:pdf},
isbn = {3540292446},
issn = {03029743},
journal = {Pkdd},
pages = {617--624},
title = {{An Imbalanced Data Rule Learner}},
year = {2005}
}
@article{Biau2015,
abstract = {The random forest algorithm, proposed by L. Breiman in 2001, has been extremely successful as a general purpose classification and re- gression method. The approach, which combines several randomized decision trees and aggregates their predictions by averaging, has shown excellent performance in settings where the number of variables is much larger than the number of observations. Moreover, it is versa- tile enough to be applied to large-scale problems, is easily adapted to various ad-hoc learning tasks, and returns measures of variable im- portance. The present article reviews the most recent theoretical and methodological developments for random forests. Emphasis is placed on the mathematical forces driving the algorithm, with special atten- tion given to the selection of parameters, the resampling mechanism, and variable importance measures. This review is intended to provide non-experts easy access to the main ideas.},
author = {Biau, G\'{e}rard and Scornet, Erwan},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Biau, Scornet - 2015 - A random forest guided tour.pdf:pdf},
title = {{A random forest guided tour}},
year = {2015}
}
@article{Xiong2014,
abstract = {Binary classification plays an important role in many decision-making processes. Random forests can build a strong ensemble classifier by combining weaker classification trees that are de-correlated. The strength and correlation among individual classification trees are the key factors that contribute to the ensemble performance of random forests. We propose roughened random forests, a new set of tools which show further improvement over random forests in binary classification. Roughened random forests modify the original dataset for each classification tree and further reduce the correlation among individual classification trees. This data modification process is composed of artificially imposing missing data that are missing completely at random and subsequent missing data imputation.},
author = {Xiong, Kuangnan},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Xiong - 2014 - Roughened Random Forests for Binary Classification.pdf:pdf},
number = {May},
title = {{Roughened Random Forests for Binary Classification}},
year = {2014}
}
@article{Ghahramani2003,
abstract = {Bayesian model averaging linearly mixes the probabilistic predictions of multiple models, each weighted by its posterior probability. This is the coherent Bayesian way of combining multiple models only under very restrictive assumptions, which we outline. We explore a general framework for Bayesian model combination (which differs from model averaging) in the context of classification. This framework explicitly models the relationship between each model's output and the unknown true label. The framework does not require that the models be probabilistic (they can even be human assessors), that they share prior information or receive the same training data, or that they be independent in their errors. Finally, the Bayesian combiner does not need to believe any of the models is in fact correct. We test several variants of this classifier combination procedure starting from a classic statistical model proposed by [1] and using MCMC to add more complex but important features to the model. Comparisons on several datasets to simpler methods like majority voting show that the Bayesian methods not only perform well but result in interpretable diagnostics on the data points and the models.},
author = {Ghahramani, Zoubin and Kim, Hyun-Chul},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Ghahramani, Kim - 2003 - Bayesian Classifier Combination.pdf:pdf},
keywords = {Computational, Information-Theoretic Learning with},
title = {{Bayesian Classifier Combination}},
url = {http://eprints.pascal-network.org/archive/00000806/},
year = {2003}
}
@book{Nielsen2015,
author = {Nielsen, Michael A.},
publisher = {Determination Press},
title = {{Neural Networks and Deep Learning}},
url = {http://neuralnetworksanddeeplearning.com/},
year = {2015}
}
@article{Zhang2009,
author = {Zhang, Aijun},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Zhang - 2009 - Statistical Methods in Credit Risk Modeling.pdf:pdf},
pages = {1--156},
title = {{Statistical Methods in Credit Risk Modeling}},
year = {2009}
}
@article{Schapire2003,
abstract = {Diversity among the members of a team of classifiers is deemed to be a key issue in classifier combination. However, measuring diversity is not straightforward because there is no generally accepted formal definition.We have found and studied ten statistics which can measure diversity among binary classifier outputs (correct or incorrect vote for the class label): four averaged pairwise measures (the Q statistic, the correlation, the disagreement and the double fault) and six non-pairwise measures (the entropy of the votes, the difficulty index, the Kohavi-Wolpert variance, the interrater agreement, the generalized diversity, and the coincident failure diversity). Four experiments have been designed to examine the relationship between the accuracy of the team and the measures of diversity, and among the measures themselves. Although there are proven connections between diversity and accuracy in some special cases, our results raise some doubts about the usefulness of diversity measures in building classifier ensembles in real-life pattern recognition problems.},
author = {Kuncheva, Ludmila I. and Whitaker, Christopher J.},
doi = {10.1049/ic:20010105},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Kuncheva, Whitaker - 2003 - Measures of Diversity in Classifier Ensembles and Their Relationship with the Ensemble Accuracy.pdf:pdf},
issn = {08856125},
journal = {Machine Learning},
keywords = {dependency and diversity,majority vote,multiple classifiers ensemble/committee of learner,pattern recognition},
number = {2},
pages = {181--207},
title = {{Measures of Diversity in Classifier Ensembles and Their Relationship with the Ensemble Accuracy}},
url = {http://citeseer.ist.psu.edu/kuncheva00measures.html},
volume = {51},
year = {2003}
}
@article{Breiman1996,
abstract = {In bagging, predictors are constructed using bootstrap samples from the training set and then aggregated to form a bagged predictor. Each bootstrap sample leaves out about 37\% of the examples. These left-out examples can be used to form accurate estimates of important quantities. For instance, they can be used to give much improved estimates of node probabilities and node error rates in decision trees. Using estimated outputs instead of the observed outputs improves accuracy in regression trees. They can also be used to give nearly optimal estimates of generalization errors for bagged predictors. * Partially supported by NSF Grant 1-444063-21445 Introduction: We assume that there is a training set T= \{(y n ,x n ), n=1, ... ,N\} and a method for constructing a predictor Q(x,T) using the given training set. The output variable y can either be a class label (classification) or numerical (regression). In bagging (Breiman[1996a]) a sequence of training sets T B,1 , ... , T B,K are generated ...},
author = {Breiman, Leo},
doi = {10.1016/j.patcog.2009.05.010},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Breiman - 1996 - Out-of-bag estimation.pdf:pdf},
issn = {00313203},
pages = {1--13},
title = {{Out-of-bag estimation}},
url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.45.3712\&rep=rep1\&type=pdf},
year = {1996}
}
@article{Scornet2014a,
abstract = {Random forests are a learning algorithm proposed by Breiman (2001) which combines several randomized decision trees and aggregates their predictions by averaging. Despite its wide usage and outstanding practical performance, little is known about the mathematical properties of the procedure. This disparity between theory and practice originates in the difficulty to simultaneously analyze both the randomization process and the highly data-dependent tree structure. In the present paper, we take a step forward in forest exploration by proving a consistency result for Breiman's (2001) original algorithm in the context of additive regression models. Our analysis also sheds an interesting light on how random forests can nicely adapt to sparsity in high-dimensional settings.},
archivePrefix = {arXiv},
arxivId = {arXiv:1405.2881v1},
author = {Scornet, Erwan and Biau, G\'{e}rard and Vert, Jean-Philippe},
eprint = {arXiv:1405.2881v1},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Scornet, Biau, Vert - 2014 - Consistency of Random Forests.pdf:pdf},
keywords = {Random forests,additive model,consistency,dimension reduction.,randomization,sparsity},
title = {{Consistency of Random Forests}},
url = {http://arxiv.org/abs/1405.2881},
year = {2014}
}
@article{Caruana2011a,
author = {Caruana, Rich and Niculescu-Mizil, Alexandru and Crew, Geoff and Ksikes, Alex},
doi = {10.1145/1015330.1015432},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Caruana et al. - 2004 - Ensemble Selection from Librairies of Models.pdf:pdf},
isbn = {1581138385},
journal = {ICML},
pages = {1--21},
title = {{Ensemble Selection from Librairies of Models}},
url = {papers://5e3e5e59-48a2-47c1-b6b1-a778137d3ec1/Paper/p1953},
volume = {34},
year = {2004}
}
@article{Monteith2011b,
abstract = {Bayesian methods are theoretically optimal in many situations. Bayesian model averaging is generally considered the standard model for creating ensembles of learners using Bayesian methods, but this technique is often out-performed by more ad hoc methods in empirical studies. The reason for this failure has important theoretical implications for our understanding of why ensembles work. It has been proposed that Bayesian model averaging struggles in practice because it accounts for uncertainty about which model is correct but still operates under the assumption that only one of them is. In order to more effectively access the benefits inherent in ensembles, Bayesian strategies should therefore be directed more towards model combination rather than the model selection implicit in Bayesian model averaging. This work provides empirical verification for this hypothesis using several different Bayesian model combination approaches tested on a wide variety of classification problems. We show that even the most simplistic of Bayesian model combination strategies outperforms the traditional ad hoc techniques of bagging and boosting, as well as outperforming BMA over a wide variety of cases. This suggests that the power of ensembles does not come from their ability to account for model uncertainty, but instead comes from the changes in representational and preferential bias inherent in the process of combining several different models.},
author = {Monteith, Kristine and Carroll, James L. and Seppi, Kevin and Martinez, Tony},
doi = {10.1109/IJCNN.2011.6033566},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Monteith et al. - 2011 - Turning Bayesian model averaging into Bayesian model combination.pdf:pdf},
isbn = {978-1-4244-9636-5},
issn = {2161-4393},
journal = {The 2011 International Joint Conference on Neural Networks},
pages = {2657--2663},
title = {{Turning Bayesian model averaging into Bayesian model combination}},
year = {2011}
}
@article{Deng2012,
archivePrefix = {arXiv},
arxivId = {arXiv:1209.6425v3},
author = {Deng, Houtao and Runger, George},
eprint = {arXiv:1209.6425v3},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Deng, Runger - 2012 - Gene Selection with Regularized Random Forest.pdf:pdf},
pages = {1--5},
title = {{Gene Selection with Regularized Random Forest}},
year = {2012}
}
@article{Snoek2012,
abstract = {Bayesian optimization is an effective methodol-ogy for the global optimization of functions with expensive evaluations. It relies on querying a dis-tribution over functions defined by a relatively cheap surrogate model. An accurate model for this distribution over functions is critical to the effectiveness of the approach, and is typically fit using Gaussian processes (GPs). However, since GPs scale cubically with the number of obser-vations, it has been challenging to handle objec-tives whose optimization requires many evalua-tions, and as such, massively parallelizing the op-timization. In this work, we explore the use of neural net-works as an alternative to GPs to model dis-tributions over functions. We show that per-forming adaptive basis function regression with a neural network as the parametric form performs competitively with state-of-the-art GP-based ap-proaches, but scales linearly with the number of data rather than cubically. This allows us to achieve a previously intractable degree of paral-lelism, which we apply to large scale hyperpa-rameter optimization, rapidly finding competitive models on benchmark object recognition tasks using convolutional networks, and image caption generation using neural language models. Proceedings of the 32 nd International Conference on Machine Learning, Lille, France, 2015. JMLR: W\&CP volume 37. Copy-right 2015 by the author(s).},
archivePrefix = {arXiv},
arxivId = {arXiv:1502.05700v2},
author = {Snoek, Jasper and Rippel, Oren and Swersky, Kevin and Kiros, Ryan and Satish, Nadathur and Sundaram, Narayanan and {Mostofa Ali Patwary MOSTOFAALIPATWARY}, Md and {Prabhat PRABHAT}, Intelcom and {Ryan Adams}, Lblgov P},
eprint = {arXiv:1502.05700v2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Snoek et al. - 2012 - Scalable Bayesian Optimization Using Deep Neural Networks.pdf:pdf},
title = {{Scalable Bayesian Optimization Using Deep Neural Networks}},
year = {2012}
}
@article{Li2012,
abstract = {Diversity among individual classifiers is recognized to play a key role in ensemble, however, few theoretical properties are known for classification. In this paper, by focusing on the popular ensemble pruning setting (i.e., combining classifier by voting and measuring diversity in pairwise manner), we present a theoretical study on the effect of diversity on the generalization performance of voting in the PAC-learning framework. It is disclosed that the diversity is closely-related to the hypothesis space complexity, and encouraging diversity can be regarded to apply regularization on ensemble methods. Guided by this analysis, we apply explicit diversity regularization to ensemble pruning, and propose the Diversity Regularized Ensemble Pruning (DREP) method. Experimental results show the effectiveness of DREP.},
author = {Li, Nan and Yu, Yang and Zhou, Zhi Hua},
doi = {10.1007/978-3-642-33460-3\_27},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Li, Yu, Zhou - 2012 - Diversity regularized ensemble pruning.pdf:pdf},
isbn = {9783642334597},
issn = {03029743},
journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
keywords = {diversity,diversity regularization,ensemble pruning},
number = {PART 1},
pages = {330--345},
title = {{Diversity regularized ensemble pruning}},
volume = {7523 LNAI},
year = {2012}
}
@article{Evans2009,
abstract = {Landscape ecology often adopts a patch mosaic model of ecological patterns. However, many ecological attributes are inherently continuous and classification of species composition into vegetation communities and discrete patches provides an overly simplistic view of the landscape. If one adopts a nichebased, individualistic concept of biotic communities then it may often be more appropriate to represent vegetation patterns as continuous measures of site suitability or probability of occupancy, rather than the traditional abstraction into categorical community types represented in a mosaic of discrete patches. The goal of this paper is to demonstrate the high effectiveness of species-level, pixel scale prediction of species occupancy as a continuous landscape variable, as an alternative to traditional classified community type vegetation maps. We use a Random Forests ensemble learning approach to predict site-level probability of occurrence for four conifer species based on climatic, topographic and spectral predictor variables across a 3,883 km2 landscape in northern Idaho, USA. Our method uses a new permutated sample-downscaling approach to equalize sample sizes in the presence and absence classes, a model selection method to optimize parsimony, and independent validation using prediction to 10\% bootstrap data withhold. The models exhibited very high accuracy, with AUC and kappa values over 0.86 and 0.95, respectively, for all four species. The spatial predictions produced by the models will be of great use to managers and scientists, as they provide vastly more accurate spatial depiction of vegetation structure across this landscape than has previously been provided by traditional categorical classified community type maps.},
author = {Evans, Jeffrey S. and Cushman, Samuel a.},
doi = {10.1007/s10980-009-9341-0},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Evans, Cushman - 2009 - Gradient modeling of conifer species using random forests.pdf:pdf},
isbn = {0921-2973},
issn = {09212973},
journal = {Landscape Ecology},
keywords = {CART,Gradient,Predictive modeling,Random forests},
number = {5},
pages = {673--683},
title = {{Gradient modeling of conifer species using random forests}},
volume = {24},
year = {2009}
}
@article{Cutler2001,
author = {Cutler, Adele and Zhao, G},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cutler, Zhao - 2001 - PERT - Perfect Random Tree Ensembles.pdf:pdf},
journal = {Computing Science and Statistics},
keywords = {boosting,classification,ensemble classifier,tree classifier},
pages = {490--497},
title = {{PERT - Perfect Random Tree Ensembles}},
url = {http://www.interfacesymposia.org/I01/I2001Proceedings/ACutler/ACutler.pdf},
volume = {33},
year = {2001}
}
@article{Cybenko,
author = {Cybenko, George},
doi = {10.1007/BF02551274},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Cybenko - 1989 - Approximation by superpositions of a sigmoidal function.pdf:pdf},
isbn = {0780300564},
issn = {0932-4194},
journal = {Mathematics of Control, Signals, and Systems},
month = dec,
number = {4},
pages = {303--314},
title = {{Approximation by superpositions of a sigmoidal function}},
url = {http://link.springer.com/10.1007/BF02551274},
volume = {2},
year = {1989}
}
@book{Rojas1996,
abstract = {Neural networks are a computing paradigm that is finding increasing attention among computer scientists. In this book, theoretical laws and models previously scattered in the literature are brought together into a general theory of artificial neural nets. Always with a view to biology and starting with the simplest nets, it is shown how the properties of models change when more general computing elements and net topologies are introduced. Each chapter contains examples, numerous illustrations, and a bibliography. The book is aimed at readers who seek an overview of the field or who wish to deepen their knowledge. It is suitable as a basis for university courses in neurocomputing.},
address = {Berlin, Heidelberg},
author = {Rojas, Ra\'{u}l},
booktitle = {Neural Networks},
doi = {10.1007/978-3-642-61068-4},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Rojas - 1996 - Neural Networks.pdf:pdf},
isbn = {978-3-540-60505-8},
number = {1},
pages = {509},
publisher = {Springer Berlin Heidelberg},
title = {{Neural Networks}},
url = {http://page.mi.fu-berlin.de/rojas/neural/index.html.html http://link.springer.com/10.1007/978-3-642-61068-4},
volume = {7},
year = {1996}
}
@inproceedings{Shi2011,
abstract = {Because the credit industry has a lot of bad debt problems, credit assessment has become a very important topic in financial institutions. Recent studies have shown that many algorithms in the fields of machine learning and artificial intelligence are competitive to statistical methods for credit assessment. Random forests, one of the most popular ensemble learning techniques, is introduced to the credit assessment problem in this paper. An experimental evaluation of different methods is carried out on the public dataset. The experimental results indicate that the random forests method improves the performance obviously.},
author = {Shi, Lei and Liu, Yi and Ma, Xinming},
booktitle = {Communications in Computer and Information Science},
doi = {10.1007/978-3-642-24282-3\_4},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Shi, Liu, Ma - 2011 - Credit assessment with random forests.pdf:pdf},
isbn = {9783642242816},
issn = {18650929},
keywords = {Credit assessment,Ensemble learning,Random forests},
pages = {24--28},
title = {{Credit assessment with random forests}},
volume = {237 CCIS},
year = {2011}
}
@article{Pozzolo2015,
abstract = {A dataset is said to be unbalanced when the class of interest (minor- ity class) is much rarer than normal behaviour (majority class). The cost of missing a minor- ity class is typically much higher that missing a majority class. Most learning sys- tems are not prepared to cope with unbalanced data. Proposed strategies essentially be- long to the following categories: sampling and distance-based. Sampling techniques up- sample or down-sample a class of instances. SMOTE generates synthetic minority exam- ples. Distance based techniques use distances between input points to under-sample or to re- move noisy and borderline examples.},
author = {Pozzolo, Andrea Dal},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Pozzolo - 2015 - Package ‘ unbalanced ’.pdf:pdf},
title = {{Package ‘ unbalanced ’}},
year = {2015}
}
@book{Vapnik1998,
author = {Vapnik, Vladimir},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Vapnik - 1998 - Statistical Learning Theory.pdf:pdf},
isbn = {978-0-471-03003-4},
title = {{Statistical Learning Theory}},
year = {1998}
}
@article{Hornik1991,
abstract = {We show that standard multilayer feedforward networks with as few as a single hidden layer and arbitrary bounded and nonconstant activation function are universal approximators with respect to Lp($\mu$) performance criteria, for arbitrary finite input environment measures $\mu$, provided only that sufficiently many hidden units are available. If the activation function is continuous, bounded and nonconstant, then continuous mappings can be learned uniformly over compact input sets. We also give very general conditions ensuring that networks with sufficiently smooth activation functions are capable of arbitrarily accurate approximation to a function and its derivatives.},
author = {Hornik, Kurt},
doi = {10.1016/0893-6080(91)90009-T},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hornik - 1991 - Approximation capabilities of multilayer feedforward networks.pdf:pdf},
isbn = {0893-6080},
issn = {08936080},
journal = {Neural Networks},
keywords = {--multilayer feedforward networks,1,activation function,approximation,d,distance between functions,environment measure,i n t r,input,measured by the uniform,o d u c,p,smooth approximation,sobolev spaces,t i o n,uniform approximation,universal approximation capabilities},
number = {2},
pages = {251--257},
title = {{Approximation capabilities of multilayer feedforward networks}},
volume = {4},
year = {1991}
}
@article{Breiman1998,
abstract = {Recent work has shown that combining multiple versions of unstable classifiers such as trees or neural nets results in reduced test set error. One of the more effective is bagging. Here, modified training sets are formed by resampling from the original training set, classifiers constructed using these training sets and then combined by voting. Freund and Schapire propose an algorithm the basis of which is to adaptively resample and combine (hence the acronym “arcing”) so that the weights in the resampling are increased for those cases most often misclassified and the combining is done by weighted voting. Arcing is more successful than bagging in test set error reduction. We explore two arcing algorithms, compare them to each other and to bagging, and try to understand how arcing works. We introduce the definitions of bias and variance for a classifier as components of the test set error. Unstable classifiers can have low bias on a large range of data sets. Their problem is high variance. Combining multiple versions either through bagging or arcing reduces variance significantly.},
author = {Breiman, Leo},
doi = {10.1214/aos/1024691079},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Breiman - 1998 - Arcing classifier (with discussion and a rejoinder by the author).pdf:pdf},
issn = {2168-8966},
journal = {The Annals of Statistics},
number = {3},
pages = {801--849},
title = {{Arcing classifier (with discussion and a rejoinder by the author)}},
volume = {26},
year = {1998}
}
@article{Harris2013,
abstract = {In this paper some of the main causes of the recent financial crisis are briefly discussed. Specific attention is paid to the accuracy of credit-scoring models used to assess consumer credit risk. As a result, the optimal default definition selection (ODDS) algorithm is proposed to improve credit-scoring for credit risk assessment. This simple algorithm selects the best default definition for use when building credit scorecards. To assess ODDS, the algorithm was used to select the default definition for the random forest tree algorithm. The resulting classification models were compared to other models built using the unselected default definitions. The results suggest that the models developed using the default definition selected by the ODDS algorithm were statistically superior to the models developed using the unselected default indicators.},
author = {Harris, Terry},
doi = {10.5430/air.v2n4p49},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Harris - 2013 - Default definition selection for credit scoring.pdf:pdf},
issn = {1927-6982},
journal = {Artificial Intelligence Research},
keywords = {credit risk,credit risk assessment,credit scoring,default definition selection,default definitions,random forest},
number = {4},
pages = {49--62},
title = {{Default definition selection for credit scoring}},
url = {http://www.sciedu.ca/journal/index.php/air/article/view/3090},
volume = {2},
year = {2013}
}
@article{Lakshminarayanan2015,
abstract = {Additive regression trees are flexible non- parametric models and popular off-the-shelf tools for real-world non-linear regression. In application domains, such as bioinformat- ics, where there is also demand for proba- bilistic predictions with measures of uncer- tainty, the Bayesian additive regression trees (BART) model, introduced by Chipman et al. (2010), is increasingly popular. As data sets have grown in size, however, the standard Metropolis–Hastings algorithms used to per- form inference in BART are proving inade- quate. In particular, these Markov chains make local changes to the trees and suffer from slow mixing when the data are high- dimensional or the best-fitting trees are more than a few layers deep. We present a novel sampler for BART based on the Particle Gibbs (PG) algorithm (Andrieu et al., 2010) and a top-down particle filtering algorithm for Bayesian decision trees (Lakshminarayanan et al., 2013). Rather than making local changes to individual trees, the PG sampler proposes a complete tree to fit the residual. Experiments show that the PG sampler out- performs existing samplers in many settings. 1},
author = {Lakshminarayanan, Balaji and Roy, Daniel M and Teh, Yee Whye},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Lakshminarayanan, Roy, Teh - 2015 - Particle Gibbs for Bayesian Additive Regression Trees.pdf:pdf},
journal = {Proceedings of the 18th International Con- ference on Artificial Intelligence and Statistics},
title = {{Particle Gibbs for Bayesian Additive Regression Trees}},
volume = {38},
year = {2015}
}
@article{Li2006,
abstract = {There has been considerable interest in random projections, an approximate algorithm for estimating distances between pairs of points in a high-dimensional vector space. Let A ∈ Rn×D be our n points in D dimensions. The method multiplies A by a random matrix R ∈ RD×k, reducing the D dimensions down to just k for speeding up the compu- tation. R typically consists of entries of standard normal N(0, 1). It is well known that random projections preserve pairwise distances (in the expectation). Achlioptas proposed sparse random projections by replacing the N(0, 1) entries in R with entries in \{−1, 0, 1\} with probabilities \{1 6 , 2 3 , 1 6\}, achieving a threefold speedup in processing time. We recommend using R of entries in \{−1, 0, 1\} with prob- abilities \{1/2√D, 1− 1/√D, 1/2√D\} for achieving a significant √D- fold speedup, with little loss in accuracy. Categories},
author = {Li, Ping and Hastie, Trevor J. and Church, Kenneth W.},
doi = {10.1145/1150402.1150436},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Li, Hastie, Church - 2006 - Very sparse random projections.pdf:pdf},
isbn = {1595933395},
journal = {Proceedings of the 12th ACM SIGKDD international conference on Knowledge discovery and data mining - KDD '06},
keywords = {expectations,matrix b preserves all,much smaller,of,pairwise dis-,provided that r consists,random projections,rates of convergence,sampling,tances of a in,the},
pages = {287},
title = {{Very sparse random projections}},
url = {http://portal.acm.org/citation.cfm?doid=1150402.1150436},
year = {2006}
}
@techreport{Durrant2014,
author = {Durrant, Robert J},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Durrant - 2014 - The Unreasonable Effectiveness of Random Projections in Computer Science.pdf:pdf},
number = {December},
title = {{The Unreasonable Effectiveness of Random Projections in Computer Science}},
year = {2014}
}
@article{Gonz2000,
abstract = {The popularity of Bayesian optimization meth- ods for efficient exploration of parameter spaces has lead to a series of papers applying Gaussian processes as surrogates in the optimization of functions. However, most proposed approaches only allowthe exploration of the parameter space to occur sequentially. Often, it is desirable to si- multaneously propose batches of parameter val- ues to explore. This is particularly the case when large parallel processing facilities are avail- able. These facilities could be computational or physical facets of the process being optimized. E.g. in biological experiments many experimen- tal set ups allow several samples to be simul- taneously processed. Batch methods, however, require modeling of the interaction between the evaluations in the batch, which can be expen- sive in complex scenarios. We investigate a sim- ple heuristic based on an estimate of the Lips- chitz constant that captures the most important aspect of this interaction (i.e. local repulsion) at negligible computational overhead. The result- ing algorithm compares well, in running time, with much more elaborate alternatives. The ap- proach assumes that the function of interest, f, is a Lipschitz continuous function. A wrap-loop around the acquisition function is used to col- lect batches of points of certain size minimizing the non-parallelizable computational effort. The speed-up of our method with respect to previous approaches is significant in a set of computation- ally expensive experiments.},
archivePrefix = {arXiv},
arxivId = {arXiv:1505.08052v2},
author = {Gonzalez, Javier and Dai, Zhenwen and Lawrence, Neil D. and Hennig, Philipp},
eprint = {arXiv:1505.08052v2},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Gonzalez et al. - 2000 - Batch Bayesian Optimization via Local Penalization.pdf:pdf},
journal = {arXiv},
number = {1},
title = {{Batch Bayesian Optimization via Local Penalization}},
year = {2000}
}
@article{Paciorek2013,
author = {Paciorek, Chris},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Paciorek - 2013 - C for statisticians , with a focus on interfacing from R and R packages.pdf:pdf},
pages = {1--60},
title = {{C ++ for statisticians , with a focus on interfacing from R and R packages}},
year = {2013}
}
@article{Hernandez,
abstract = {Bayesian Additive Regression Trees (BART) tend do be slow to converge in the case where there are far more variables than observations, as is typically the case with proteomic biomarker discovery data. Here we propose adapting BART by applying a version of Bayesian model averaging called Occam’s Window (Maddigan and Raftery, 1994). To date we have implemented a version of our algorithm which we call BART BMA in C++ and have found that there are gains in both speed and model parsimony when compared to its main competitor random forest when the BART BMA model is tuned correctly.},
author = {Hernandez, Belinda and Raftery, Adrian E and Pennington, Stephen R and Parnell, Andrew C.},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Hernandez et al. - Unknown - BART BMA Bayesian Additive Regression Trees using Bayesian Model Averaging for discovery of protein biomar.pdf:pdf},
number = {2010},
pages = {3--4},
title = {{BART BMA : Bayesian Additive Regression Trees using Bayesian Model Averaging for discovery of protein biomarkers}}
}
@article{Simpson2013a,
abstract = {Classifier combination methods need to make best use of the outputs of multiple, imperfect classifiers to enable higher accuracy classifications. In many situations, such as when human decisions need to be combined, the base decisions can vary enormously in reliability. A Bayesian approach to such uncertain combination allows us to infer the differences in performance between individuals and to incorporate any available prior knowledge about their abilities when training data is sparse. In this paper we explore Bayesian classifier combination, using the computationally efficient framework of variational Bayesian inference. We apply the approach to real data from a large citizen science project, Galaxy Zoo Supernovae, and show that our method far outperforms other established approaches to imperfect decision combination. We go on to analyse the putative community structure of the decision makers, based on their inferred decision making strategies, and show that natural groupings are formed. Finally we present a dynamic Bayesian classifier combination approach and investigate the changes in base classifier performance over time.},
archivePrefix = {arXiv},
arxivId = {1206.1831},
author = {Simpson, Edwin and Roberts, Stephen and Psorakis, Ioannis and Smith, Arfon},
doi = {10.1007/978-3-642-36406-8-1},
eprint = {1206.1831},
file = {:Users/guillaume/Dropbox/Mendeley Desktop/Simpson et al. - 2013 - Dynamic bayesian combination of multiple imperfect classifiers.pdf:pdf},
isbn = {9783642364051},
issn = {1860949X},
journal = {Studies in Computational Intelligence},
pages = {1--35},
title = {{Dynamic bayesian combination of multiple imperfect classifiers}},
volume = {474},
year = {2013}
}