\documentclass{beamer}
\usepackage{hyperref}
% \usetheme{default}
\usetheme[secheader]{Boadilla}
%\usetheme{Hannover}
\usepackage[all]{xy}
\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage{latexsym, epsfig, subfigure , color, amsmath, bbm}
\DeclareMathOperator*{\argmin}{arg\,min}
\setbeamertemplate{blocks}[rounded][shadow]
%\setbeamercovered{transparent}
%\setbeamertemplate{navigation symbols}{}
\newcommand{\omalpha}{\ensuremath{(1 - \alpha)}\,}
\newcommand{\link}[1]{\underline{\color{red}#1}}
\newcommand{\hilite}[1]{{\color{red}#1}}
\newcommand{\Ev}{\ensuremath{\mbox{Ev}}}
\newcommand{\boost}{\vphantom{\big(}}
\newenvironment{variableblock}[3]{%
  \setbeamercolor{block body}{#2}
  \setbeamercolor{block title}{#3}
  \begin{block}{#1}}{\end{block}}
\definecolor{light-blue}{rgb}{0.29, 0.59, 0.82}
\title{Statistical Inference \\ Lecture One}
\subtitle{\href{https://people.bath.ac.uk/masss/APTS/2022-23/LectureOne.pdf}{https://people.bath.ac.uk/masss/APTS/2022-23/LectureOne.pdf}}
\author{Simon Shaw}
\institute{University of Bath}
\date{APTS, 13-16 December 2022}

\begin{document}
\begin{frame}
\titlepage
\end{frame}
\section{Principles for Statistical Inference}
\subsection{Overview of Lecture One}
\begin{frame}
  \frametitle{Overview of Lecture One}
  \begin{itemize}
  \item We wish to consider inferences about a parameter \textcolor{red}{$\theta$} given a parametric model  $\textcolor{red}{\mathcal{E} \ = \ \{ \mathcal{X}, \Theta, f_{X}(x \, | \, \theta) \}}$
\begin{displaymath}
\begin{array}{c}
\xymatrix{
  (\mathcal{E}, x) \ar@{|->}[rrr]^{\txt{\boost\hspace{-7ex}statistician, $\Ev$}} &&& \text{Inference about $\theta$.} \\
}
\end{array}
\end{displaymath}
\item We'll consider a series of \textcolor{blue}{statistical principles} to guide the way to learn about $\theta$. 
\item \textcolor{blue}{Weak Indifference Principle, WIP}: if \textcolor{red}{$f_{X}(x \, | \, \theta) = f_{X}(x' \, | \, \theta)$} for all $\theta \in \Theta$ then \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}, x')$}.
\item \textcolor{blue}{Distribution Principle, DP}: if \textcolor{red}{$\mathcal{E} = \mathcal{E}'$}, then \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}', x)$}.
\item \textcolor{blue}{Transformation Principle, TP}: for the bijective \textcolor{red}{$g : \mathcal{X} \to \mathcal{Y}$}, construct \textcolor{red}{$\mathcal{E}^g = \{\mathcal{Y}, \Theta, f_{Y}(y \, | \,\theta)\}$}. Then  \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}^g, g(x))$}.
\item $(\mbox{DP} \wedge \mbox{TP} \, ) \rightarrow \mbox{WIP}.$
\end{itemize}
\end{frame}
\begin{frame}
  \frametitle{Overview of Lecture One continued}
  \begin{itemize}
\item \textcolor{blue}{Weak Conditionality Principle, WCP}: if $\mathcal{E}^{*}$ is the mixture of the experiments $\mathcal{E}_{1}$, $\mathcal{E}_{2}$ according to mixture probabilities $p_{1}$, $p_{2} = 1-p_{1}$. then \textcolor{red}{$\Ev\left(\mathcal{E}^*, (i, x_i) \right) = \Ev(\mathcal{E}_i, x_i)$}.
  \item \textcolor{blue}{Strong Likelihood Principle, SLP}: if \textcolor{red}{$f_{X_{1}}(x_{1} \, | \, \theta) = c(x_{1}, x_{2})f_{X_{2}}(x_{2} \, | \, \theta)$}, for some function $c > 0$ for all $\theta \in \Theta$ then $\textcolor{red}{\Ev(\mathcal{E}_1, x_1) = \Ev(\mathcal{E}_2, x_2)}$.
  \item \textcolor{blue}{Birnbaum's Theorem}: $(\mbox{WIP} \wedge \mbox{WCP} \, ) \leftrightarrow \mbox{SLP}.$
  \item  \textcolor{blue}{Stopping Rule Principle, SRP}: in a sequential experiment $\mathcal{E}^\tau$, $\Ev \left( \mathcal{E}^\tau, (x_1, \dots, x_n) \right)$ does not depend on the stopping rule $\tau$.
  \item $\mbox{SLP} \rightarrow \mbox{SRP}$.
\end{itemize}
\end{frame}
\subsection{Introduction}
\begin{frame}
  \frametitle{Introduction}
  \begin{itemize}
  \item We wish to consider inferences about a parameter \textcolor{red}{$\theta$} given a parametric model
    \begin{eqnarray*}
  \textcolor{red}{\mathcal{E} \ = \ \{ \mathcal{X}, \Theta, f_{X}(x \, | \, \theta) \}}.
    \end{eqnarray*}
    \item We assume that the model is \textcolor{blue}{true} so that only \textcolor{red}{$\theta \in \Theta$} is unknown. We wish to learn about \textcolor{red}{$\theta$} from observations \textcolor{red}{$x$} (typically, \textcolor{blue}{vector valued}) so that \textcolor{red}{$\mathcal{E}$} represents a model for this \textcolor{blue}{experiment}.
    \end{itemize}
\ \newline
  Smith (2010) considers that there are \textcolor{blue}{three} players in an inference problem:
  \begin{enumerate}
  \item \textcolor{blue}{Client:}  person with the problem
  \item \textcolor{blue}{Statistician:} employed by the client to help solve the problem
  \item \textcolor{blue}{Auditor:} hired by the client to check the statistician's work
  \end{enumerate}
  The statistician is thus responsible for explaining the rationale behind the choice of inference in a compelling way.
\end{frame}
\subsection{Reasoning about inferences}
\begin{frame}
  \frametitle{Reasoning about inferences}
  We consider a series of \textcolor{blue}{statistical principles} to guide the way to learn about $\theta$. The principles are meant to be either \textcolor{blue}{self-evident} or \textcolor{blue}{logical implications} of principles which are self-evident.
  \ \newline
 We shall assume that $\mathcal{X}$ is \textcolor{blue}{finite}: Basu (1975) argues that ``infinite and continuous models are to be looked upon as mere approximations to the finite realities.''

  \begin{itemize}
  \item Inspiration of \href{https://en.wikipedia.org/wiki/Allan_Birnbaum}{\textcolor{blue}{\underline{Allan Birnbaum (1923-1976)}}} to see how to construct and reason about statistical principles given ``\textcolor{blue}{evidence}'' from data.
   \item The model \textcolor{red}{$\mathcal{E} = \{ \mathcal{X}, \Theta, f_{X}(x \, | \, \theta) \}$} is accepted as a working hypothesis.
   \item How the statistician chooses her inference statements about the true value $\theta$ is entirely down to her and her client.
     \begin{itemize}
     \item as a point or a set in $\Theta$;
     \item as a choice among alternative sets or actions;
      \item or maybe as something more complicated, not ruling out visualisations.
     \end{itemize}
\end{itemize}
     \end{frame}
\begin{frame}
  \frametitle{\ }
  \begin{itemize}
   \item Following Dawid (1977), consider that the statistician defines, \textcolor{blue}{a priori}, a set of possible \textcolor{blue}{inferences about $\theta$}
   \item Task is to choose an element of this set based on \textcolor{red}{$\mathcal{E}$} and \textcolor{red}{$x$}.
   \item  The statistician should see herself as a function  \textcolor{red}{$\Ev$}: a mapping from  \textcolor{red}{$(\mathcal{E}, x)$} into a predefined set of \textcolor{blue}{inferences about~$\theta$}.
   \end{itemize}
\begin{displaymath}
\begin{array}{c}
\xymatrix{
  (\mathcal{E}, x) \ar@{|->}[rrr]^{\txt{\boost\hspace{-7ex}statistician, $\Ev$}} &&& \text{Inference about $\theta$.} \\
}
\end{array}
\end{displaymath}
\begin{itemize}
\item For example, \textcolor{red}{$\Ev(\mathcal{E}, x)$} might be:
\begin{itemize}
\item the maximum likelihood estimator of $\theta$
\item a 95\% confidence interval for $\theta$
\end{itemize}
\item Birnbaum called \textcolor{red}{$\mathcal{E}$} the \textcolor{blue}{experiment}, \textcolor{red}{$x$} the \textcolor{blue}{outcome}, and \textcolor{red}{$\Ev$} the \textcolor{blue}{evidence}.
\end{itemize}
\end{frame}
\begin{frame}
  \frametitle{\ }
  Note:
  \begin{enumerate}
  \item There can be \textcolor{blue}{different} experiments with the same $\theta$.
  \item Under some outcomes, we would agree that it is self-evident that these different experiments provide the \textcolor{blue}{same evidence} about $\theta$.
  \end{enumerate}
  \begin{block}{Example}
    Consider two experiments with the same $\theta$.
    \begin{enumerate}
    \item $X \sim Bin(n, \theta)$, so we observe $x$ successes in $n$ trials.
    \item $Y \sim NBin(r, \theta)$, so we observe the $r$th success in the $y$th trial.
    \end{enumerate}
    If we observe $x = r$ and $y = n$, do we make the same inference about $\theta$ in each case?
    \end{block}
  \end{frame}
  \begin{frame}
    \frametitle{\ }
 
 Consider two experiments \textcolor{red}{$\mathcal{E}_{1} = \{\mathcal{X}_{1}, \Theta, f_{X_{1}}(x_{1} \, | \, \theta)\}$} and  \textcolor{red}{$\mathcal{E}_{2} = \{\mathcal{X}_{2}, \Theta, f_{X_{2}}(x_{2} \, | \, \theta)\}$}.
 \begin{block}{Equivalence of evidence (Basu, 1975)}
   The \textcolor{blue}{equality} or  \textcolor{blue}{equivalence} of  \textcolor{red}{$\Ev(\mathcal{E}_{1}, x_{1})$} and  \textcolor{red}{$\Ev(\mathcal{E}_{2}, x_{2})$} means that:
   \begin{enumerate}
   \item \textcolor{red}{$\mathcal{E}_{1}$} and  \textcolor{red}{$\mathcal{E}_{2}$} are related to the  \textcolor{blue}{same} parameter $\theta$.
   \item \textcolor{blue}{Everything else being equal}, the outcome  \textcolor{red}{$x_{1}$} from  \textcolor{red}{$\mathcal{E}_{1}$}  \textcolor{blue}{warrants the same inference} about $\theta$ as does the outcomes  \textcolor{red}{$x_{2}$} from  \textcolor{red}{$\mathcal{E}_{2}$}.
   \end{enumerate}
 \end{block}
 \begin{itemize}
\item We now consider constructing statistical principles and demonstrate how these principles imply other principles.
\item These principles all have the same form: under such and such conditions, the evidence about $\theta$ should be the same.
\item Thus they serve only to rule out inferences that satisfy the conditions but have different evidences. They do not tell us how to do an inference, only what to avoid.
\end{itemize}
\end{frame}
\subsection{The principle of indifference}
 \begin{frame}
   \frametitle{The principle of indifference}
   \begin{block}{Principle 1: Weak Indifference Principle, WIP}
     Let $\mathcal{E} = \{ \mathcal{X}, \Theta, f_{X}(x \, | \, \theta) \}$.  If \textcolor{red}{$f_{X}(x \, | \, \theta) = f_{X}(x' \, | \, \theta)$} for all $\theta \in \Theta$ then \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}, x')$}.
     \end{block}
     \begin{itemize}
     \item We are indifferent between two models of evidence if they differ only in the manner of the labelling of sample points.
     \item If $X = (X_{1}, \ldots, X_{n})$ where the $X_{i}$s are a series of independent Bernoulli trials with parameter $\theta$ then $f_{X}(x \, | \, \theta) = f_{X}(x' \, | \, \theta)$ if $x$ and $x'$ contain the same number of successes.
       \end{itemize}
     \end{frame}
\begin{frame}
   %\frametitle{\ }
   \begin{block}{Principle 2: Distribution Principle, DP}
     If \textcolor{red}{$\mathcal{E} = \mathcal{E}'$}, then \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}', x)$}.
   \end{block}
   \begin{itemize}
   \item Informally, (Dawid, 1977), only aspects of an experiment which are relevant to inference are the sample space and the family of distributions over it.
     \end{itemize}
   \begin{block}{Principle 3: Transformation Principle, TP}
     Let $\mathcal{E} = \{\mathcal{X}, \Theta, f_{X}(x \, | \,\theta)\}$.  For the bijective \textcolor{red}{$g : \mathcal{X} \to \mathcal{Y}$}, let $\mathcal{E}^g = \{\mathcal{Y}, \Theta, f_{Y}(y \, | \,\theta)\}$, the \textcolor{blue}{same} experiment as $\mathcal{E}$  but expressed in terms of \textcolor{red}{$Y = g(X)$}, rather than $X$. Then \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}^g, g(x))$}.
     \end{block}
\begin{itemize}
\item Inferences should not depend on the way in which the sample space is labelled, for example, $X$ or $X^{-1}$.
  \end{itemize}
\end{frame}
\begin{frame}
  % \frametitle{\ }
  \begin{block}{Theorem}
    $(\mbox{DP} \wedge \mbox{TP} \, ) \rightarrow \mbox{WIP}.$
  \end{block}
  \begin{block}{Proof}
    Fix $\mathcal{E}$, and suppose that $x, x' \in \mathcal{X}$ satisfy \textcolor{red}{$f_{X}(x \, | \, \theta) = f_{X}(x' \, | \, \theta)$} for all $\theta \in \Theta$, as in the condition of the WIP.

    Let $g : \mathcal{X} \to \mathcal{X}$ be the function which \textcolor{blue}{switches} $x$ for $x'$,  but leaves all of the other elements of $\mathcal{X}$ \textcolor{blue}{unchanged}. Then \textcolor{red}{$\mathcal{E} = \mathcal{E}^g$} and
    \begin{eqnarray*}
  \Ev(\mathcal{E}, x')
  & = & \Ev(\mathcal{E}^g, x')  \ \ \mbox{  [by the DP]}\\
  & = & \Ev(\mathcal{E}^g, g(x)) \\\
  & = & \Ev(\mathcal{E}, x),  \ \ \mbox{  [by the TP]}
    \end{eqnarray*}
    which gives the WIP. \hfill $\Box$
  \end{block}
\end{frame}
\subsection{The Likelihood Principle}
\begin{frame}
  \frametitle{The Likelihood Principle}
  \begin{itemize}
  \item Consider experiments \textcolor{red}{$\mathcal{E}_{i} = \{\mathcal{X}_{i}, \Theta, f_{X_{i}}(x_{i} \, | \, \theta)\}$}, $i = 1, 2, \ldots$, where the parameter space $\Theta$ is the same for each experiment.
  \item Let \textcolor{red}{$p_{1}, p_{2}, \ldots$} be a set of known \textcolor{blue}{probabilities} so that $p_{i} \geq 0$ and $\sum_{i} p_{i} = 1$.
  \end{itemize}
  \begin{block}{Mixture experiment}
    The \textcolor{blue}{mixture} \textcolor{red}{$\mathcal{E}^{*}$} of the experiments $\mathcal{E}_{1}, \mathcal{E}_{2}, \ldots$ according to mixture probabilities $p_{1}, p_{2}, \ldots$ is the \textcolor{blue}{two-stage experiment}
\begin{enumerate}
\item A random selection of one of the experiments: $\mathcal{E}_{i}$ is selected with probability $p_{i}$.
\item The experiment selected in stage 1. is performed.
\end{enumerate} 
Thus, each \textcolor{blue}{outcome} of the experiment $\mathcal{E}^{*}$ is a pair \textcolor{red}{$(i, x_{i})$}, where $i = 1, 2, \ldots$ and $x_{i} \in \mathcal{X}_{i}$, and family of distributions
\begin{eqnarray*}
\textcolor{red}{f^{*}((i, x_{i}) \, | \, \theta) \ = \ p_{i}f_{X_{i}}(x_{i} \, | \, \theta)}. 
\end{eqnarray*}
\end{block}
\end{frame}
\begin{frame}
  \frametitle{\ }
  \begin{block}{Principle 4: Weak Conditionality Principle, WCP}
    Let \textcolor{red}{$\mathcal{E}^{*}$} be the mixture of the experiments \textcolor{red}{$\mathcal{E}_{1}$, $\mathcal{E}_{2}$} according to mixture probabilities $p_{1}$, $p_{2} = 1-p_{1}$. Then \textcolor{red}{$\Ev\left(\mathcal{E}^*, (i, x_i) \right) = \Ev(\mathcal{E}_i, x_i)$}.
  \end{block}
\begin{itemize}
\item The WCP says that inferences for $\theta$ depend \textcolor{blue}{only} on the experiment performed and not which experiments \textcolor{blue}{could have} been performed.
 \item Suppose that \textcolor{red}{$\mathcal{E}_i$} is \textcolor{blue}{randomly} chosen with probability \textcolor{red}{$p_{i}$} and \textcolor{red}{$x_{i}$} is observed.
 \item The WCP states that the \textcolor{blue}{same evidence} about $\theta$ would have been obtained if it was decided \textcolor{blue}{non-randomly} to perform \textcolor{red}{$\mathcal{E}_i$} from the \textcolor{blue}{beginning} and \textcolor{red}{$x_{i}$} is observed.
   \end{itemize}
  \end{frame}
  \begin{frame}
  \frametitle{\ }
  \begin{block}{Principle 5: Strong Likelihood Principle, SLP}
    Let $\mathcal{E}_1$ and $\mathcal{E}_2$ be two experiments which have the same parameter $\theta$.  If $x_1 \in \mathcal{X}_1$ and $x_2 \in \mathcal{X}_2$ satisfy \textcolor{red}{$f_{X_{1}}(x_{1} \, | \, \theta) = c(x_{1}, x_{2})f_{X_{2}}(x_{2} \, | \, \theta)$}, that is
\begin{eqnarray*}\label{eq:LP}
 \textcolor{red}{L_{X_{1}}(\theta; x_{1}) \ = \ c(x_{1}, x_{2})L_{X_{2}}(\theta; x_{2})}
\end{eqnarray*}
for some function $c > 0$ for all $\theta \in \Theta$ then $\textcolor{red}{\Ev(\mathcal{E}_1, x_1) = \Ev(\mathcal{E}_2, x_2)}$.
\end{block}
\begin{itemize}
\item The SLP states that if two likelihood functions for the same parameter have the same shape, then the evidence is the same.
 \item A corollary of the SLP, obtained by setting $\mathcal{E}_1 = \mathcal{E}_2 = \mathcal{E}$, is that $\Ev(\mathcal{E}, x)$ should depend on $\mathcal{E}$ and $x$ only through $L_{X}(\theta; x)$.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
Many classical statistical procedures violate the SLP and the following result was something of the bombshell, when it first emerged in the 1960s.  The following form is due to Birnbaum (1972) and Basu (1975)
\begin{block}{Birnbaum's Theorem}
  $(\mbox{WIP} \wedge \mbox{WCP} \, ) \leftrightarrow \mbox{SLP}.$
\end{block}
\begin{block}{Proof}
  Both $\mbox{SLP} \rightarrow \mbox{WIP}$ and $\mbox{SLP} \rightarrow  \mbox{WCP}$ are straightforward.  The trick is to prove $(\mbox{WIP}
  \wedge \mbox{WCP} \,) \rightarrow \mbox{SLP}$.

  Let \textcolor{red}{$\mathcal{E}_1$} and  \textcolor{red}{$\mathcal{E}_2$} be two experiments which have the same parameter, and suppose that  \textcolor{red}{$x_1 \in \mathcal{X}_1$} and  \textcolor{red}{$x_2 \in \mathcal{X}_2$} satisfy  \textcolor{red}{$f_{X_{1}}(x_{1} \, | \, \theta) = c(x_{1}, x_{2})f_{X_{2}}(x_{2} \, | \, \theta)$} where the function  \textcolor{red}{$c > 0$}. As the value  \textcolor{red}{$c$} is  \textcolor{blue}{known} (as the data has been observed) then consider the  \textcolor{blue}{mixture experiment} with  \textcolor{red}{$p_{1} = 1/(1+c)$} and  \textcolor{red}{$p_{2} = c/(1+c)$}.
\end{block}
\end{frame}
\begin{frame}
  % \frametitle{\ }
\begin{block}{Proof continued}
\begin{eqnarray*}
f^{*}((1, x_{1}) \, | \, \theta) \ = \ \frac{1}{1+c} \textcolor{red}{f_{X_{1}}(x_{1} \, | \, \theta)}  \ = \ \frac{\textcolor{red}{c}}{1+c} \textcolor{red}{f_{X_{2}}(x_{2} \, | \, \theta)}  \ = \ f^{*}((2, x_{2}) \, | \, \theta)
\end{eqnarray*}
Then the \textcolor{blue}{WIP} implies that
\begin{eqnarray*}
  \Ev \left( \mathcal{E}^*, (1, x_1) \right) & = & \Ev \left( \mathcal{E}^*, (2, x_2) \right).
\end{eqnarray*}
Applying the \textcolor{blue}{WCP} to each side we infer that
\begin{eqnarray*}
  \Ev(\mathcal{E}_1, x_1) & = & \Ev(\mathcal{E}_2, x_2) ,
\end{eqnarray*}
as required.  \hfill $\Box$
\end{block}
Thus, either I accept the SLP, or I explain which of the two principles, WIP and WCP, I refute. Methods, which include many  \textcolor{blue}{classical procedures}, which violate the SLP face exactly this challenge.
\end{frame}
\subsection{The Sufficiency Principle} 
\begin{frame}
\frametitle{The Sufficiency Principle}
\begin{itemize}
\item Recall the idea of sufficiency: if \textcolor{red}{$S = s(X)$} is \textcolor{blue}{sufficient} for \textcolor{red}{$\theta$} then 
\begin{eqnarray*}
\textcolor{red}{f_{X}(x \, | \, \theta)} & = & f_{X | S}(x \, | \, s, \theta)\textcolor{red}{f_{S}(s \, | \, \theta)} \label{eq:suff1}
\end{eqnarray*}
where $f_{X | S}(x \, | \, s, \theta)$ does not depend upon $\theta$. 
\item Consequently, consider the experiment \textcolor{red}{$\mathcal{E}^{S} = \{s(\mathcal{X}), \Theta, f_{S}(s \, | \, \theta)\}$}. 
\end{itemize}
\begin{block}{Principle 6: Strong Sufficiency Principle, SSP}
If \textcolor{red}{$S = s(X)$} is a sufficient statistic for $\mathcal{E} = \{\mathcal{X}, \Theta, f_{X}(x \, | \, \theta)\}$ then \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}^{S}, s(x))$}.
\end{block}
\begin{block}{Principle 7: Weak Sufficiency Principle, WSP}
If \textcolor{red}{$S = s(X)$} is a sufficient statistic for $\mathcal{E} = \{\mathcal{X}, \Theta, f_{X}(x \, | \, \theta)\}$ and \textcolor{red}{$s(x) = s(x')$} then \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}, x')$}.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Theorem}
$\mbox{SLP} \rightarrow \mbox{SSP} \rightarrow \mbox{WSP} \rightarrow \mbox{WIP}$.
\end{block}
\begin{block}{Proof} 
As $s$ is \textcolor{blue}{sufficient}, \textcolor{red}{$f_{X}(x \, | \, \theta) = cf_{S}(s \, | \, \theta)$} where $c = f_{X | S}(x \, | \, s, \theta)$ does not depend on $\theta$. Applying the \textcolor{blue}{SLP}, \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}^{S}, s(x))$} which is the \textcolor{blue}{SSP}.

Note, that from the \textcolor{blue}{SSP},
\begin{eqnarray*}
\Ev(\mathcal{E}, x) & = & \Ev(\mathcal{E}^{S}, s(x)) \ \ \ \ \ \mbox{(by the \textcolor{blue}{SSP})}\\
& = & \Ev(\mathcal{E}^{S}, s(x')) \ \ \ \ \mbox{(as \textcolor{red}{$s(x) = s(x')$})}\\
& = & \Ev(\mathcal{E}, x') \ \ \ \ \ \ \, \ \ \ \mbox{(by the \textcolor{blue}{SSP})}
\end{eqnarray*}
We thus have the \textcolor{blue}{WSP}. Finally, if \textcolor{red}{$f_{X}(x \, | \, \theta) = f_{X}(x' \, | \, \theta)$} as in the statement of \textcolor{blue}{WIP} then \textcolor{red}{$s(x) = x'$} is \textcolor{blue}{sufficient} for \textcolor{red}{$x$}. Hence, from the \textcolor{blue}{WSP},  \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}, x')$} giving the \textcolor{blue}{WIP}. \hfill $\Box$
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
If we put together the last two theorems, we get the following corollary.
\begin{block}{Corollary}
 $(\mbox{WIP} \wedge  \mbox{WCP}) \rightarrow \mbox{SSP}$.
\end{block}
\begin{block}{Proof}
From Birnbaum's theorem, $(\mbox{WIP} \wedge \mbox{WCP} \, ) \leftrightarrow \mbox{SLP}$ and from the previous theorem, $\mbox{SLP} \rightarrow \mbox{SSP}$. \hfill $\Box$
\end{block}
\begin{itemize}
\item Birnbaum's (1962) original result combined \textcolor{blue}{sufficiency} and \textcolor{blue}{conditionality} for the \textcolor{blue}{likelihood} but he revised this to the \textcolor{blue}{WIP} and \textcolor{blue}{WCP} in later work.
\item One advantage of this is that it reduces the dependency on sufficiency: \textcolor{blue}{Pitman-Koopman-Darmois Theorem} states that sufficiency more-or-less characterises the \textcolor{blue}{exponential family}.
\end{itemize}
\end{frame}
\subsection{Stopping rules}
\begin{frame}
\frametitle{Stopping rules}
\begin{itemize}
\item Consider observing a sequence of random variables \textcolor{red}{$X_{1}, X_{2}, \ldots$} where the number of observations is \textcolor{blue}{not fixed in advance} but depends on the values seen so far. 
\begin{itemize}
\item At time \textcolor{red}{$j$}, the decision to observe \textcolor{red}{$X_{j+1}$} can be modelled by a probability \textcolor{red}{$p_j(x_1, \dots, x_j)$}. 
\item We assume, resources being finite, that the experiment \textcolor{blue}{must stop} at specified time \textcolor{red}{$m$}, if it has not stopped already, hence \textcolor{red}{$p_m(x_1, \dots, x_m) = 0$}. 
\end{itemize}
\item The \textcolor{blue}{stopping rule} may then be denoted as \textcolor{red}{$\tau = (p_1, \dots, p_m)$}. This gives an experiment \textcolor{red}{$\mathcal{E}^{\tau}$} with, for $n = 1, 2, \ldots$, \textcolor{red}{$f_n(x_1, \dots, x_n \, | \, \theta)$} where consistency requires that
\end{itemize}
\begin{eqnarray*}
  f_n(x_1, \dots, x_n \, | \, \theta) & = & \sum_{x_{n+1}} \cdots \sum_{x_m} f_{m}(x_1, \dots, x_n, x_{n+1}, \dots x_m \, | \, \theta).
\end{eqnarray*}
\end{frame} 
\begin{frame}
\frametitle{Motivation for the stopping rule principle (Basu, 1975)}
\begin{itemize}
\item Consider four \textcolor{blue}{different} coin-tossing experiments (with some finite limit on the number of tosses).
\begin{enumerate}
\item[\textcolor{red}{$\mathcal{E}_1$}] Toss the coin exactly $10$ times; 
\item[\textcolor{red}{$\mathcal{E}_2$}] Continue tossing until $6$ heads appear;
\item[\textcolor{red}{$\mathcal{E}_3$}] Continue tossing until $3$ consecutive heads appear;
\item[\textcolor{red}{$\mathcal{E}_4$}] Continue tossing until the accumulated number of heads exceeds that of tails by exactly $2$.
\end{enumerate}
\item Suppose that all four experiments have the \textcolor{blue}{same outcome} ${x = (\mbox{T,H,T,T,H,H,T,H,H,H})}$.
\item We may feel that the evidence for $\theta$, the probability of heads, is the \textcolor{blue}{same in every case}. 
\begin{itemize}
\item Once the sequence of heads and tails is known, the intentions of the original experimenter (i.e.\ the experiment she was doing) are \textcolor{blue}{immaterial to inference} about the probability of heads.
\item The simplest experiment $\mathcal{E}_1$ can be used for inference. 
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Principle 8: Stopping Rule Principle, SRP}\footnote{Basu (1975) claims the SRP is due to \href{https://en.wikipedia.org/wiki/George_Alfred_Barnard}{\textcolor{light-blue}{\underline{George Barnard (1915-2002)}}}}
  In a sequential experiment $\mathcal{E}^\tau$, $\Ev \left( \mathcal{E}^\tau, (x_1, \dots, x_n) \right)$ does not depend on the stopping rule $\tau$.
\end{block}
\begin{itemize}
\item If it is accepted, the SRP is nothing short of revolutionary. 
\item It implies that the \textcolor{blue}{intentions} of the experimenter, represented by \textcolor{red}{$\tau$}, are \textcolor{blue}{irrelevant} for making inferences about \textcolor{red}{$\theta$}, once the observations \textcolor{red}{$(x_1, \dots, x_n)$} are \textcolor{blue}{known}. 
\item Once the data is \textcolor{blue}{observed}, we can \textcolor{blue}{ignore} the sampling plan.  
\item The statistician could proceed as though the \textcolor{blue}{simplest possible stopping rule }were in effect, which is \textcolor{red}{$p_1 = \cdots = p_{n-1} = 1$} and \textcolor{red}{$p_n = 0$}, an experiment with \textcolor{red}{$n$} \textcolor{blue}{fixed in advance}, \textcolor{red}{$\mathcal{E}^{n} = \{\mathcal{X}_{1:n}, \Theta, f_{n}(x_{1: n} \, | \, \theta)\}$}.
\item Can the SRP possibly be justified?  Indeed it can.
\end{itemize} 
\end{frame}
\begin{frame}
%\frametitle{\ } 
\begin{block}{Theorem}
  $\mbox{SLP} \rightarrow \mbox{SRP}$.
\end{block} 
\begin{block}{Proof} Let $\tau$ be an arbitrary stopping rule, and consider the outcome $(x_1, \dots, x_n)$, which we will denote as $x_{1:n}$. 
\begin{itemize}
\item We \textcolor{blue}{take} the \textcolor{red}{first} observation with probability \textcolor{red}{one}.
\item For $j=1, \ldots, n-1$, the \textcolor{red}{$(j+1)$}th observation is \textcolor{blue}{taken} with probability \textcolor{red}{$p_{j}(x_{1:j})$}.
\item We \textcolor{blue}{stop} after the \textcolor{red}{$n$}th observation with probability \textcolor{red}{$1 - p_n(x_{1:n})$}. 
\end{itemize}
Consequently, the probability of this outcome under $\tau$ is
\begin{eqnarray*}\
f_\tau(x_{1:n} \, | \, \theta)  &  = & f_1(x_1 \, | \, \theta) \left\{
    \prod_{j=1}^{n-1} \textcolor{red}{p_j(x_{1:j})} \, f_{j+1}(x_{j+1} \, | \, x_{1:j}, \theta)
    \right\} \textcolor{red}{\left( 1 - p_n(x_{1:n}) \right)}
\end{eqnarray*}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }  
\begin{block}{Proof continued}    
\begin{eqnarray*}
 f_\tau(x_{1:n} \, | \, \theta)    
  & = & \left\{\prod_{j=1}^{n-1} p_j(x_{1:j}) \right\} \left( 1 - p_n(x_{1:n}) \right)
  \textcolor{red}{f_1(x_1 \, | \, \theta) \prod_{j=2}^n f_j(x_j \, | \,  x_{1:(j-1)}, \theta)} \\
  & = & \textcolor{blue}{\left\{\prod_{j=1}^{n-1} p_j(x_{1:j}) \right\} \left( 1 - p_n(x_{1:n}) \right)} \textcolor{red}{f_n(x_{1:n} \, | \, \theta)}.
\end{eqnarray*}
Now observe that this equation has the form
\begin{eqnarray}
  f_\tau(x_{1:n} \, | \, \theta) & = & \textcolor{blue}{c(x_{1:n})}f_n(x_{1:n} \, | \,  \theta) \label{ex:niceSRP}
\end{eqnarray}
where $c(x_{1:n}) > 0$. Thus the \textcolor{blue}{SLP} implies that \textcolor{red}{$\Ev(\mathcal{E}^\tau, x_{1:n}) = \Ev(\mathcal{E}^n, x_{1:n})$} where \textcolor{red}{$\mathcal{E}^n = \{ \mathcal{X}_{1: n}, \Theta, f_n(x_{1:n} \, | \, \theta) \}$}.  Since the choice of stopping rule was \textcolor{blue}{arbitrary}, equation (\ref{ex:niceSRP}) holds \textcolor{blue}{for all} stopping rules, showing that the choice of stopping rule is \textcolor{blue}{irrelevant}.  \hfill $\Box$
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
A comment from \href{https://en.wikipedia.org/wiki/Leonard_Jimmie_Savage}{\textcolor{light-blue}{\underline{Leonard Jimmie Savage (1917-1971)}}}, one of the great statisticians of the Twentieth Century, captured the \textcolor{blue}{revolutionary} and \textcolor{blue}{transformative nature} of the SRP.
\begin{quote}
  May I digress to say publicly that I learned the stopping rule
  principle from Professor Barnard, in conversation in the summer of
  1952.  Frankly, I then thought it \textcolor{blue}{a scandal} that anyone in the
  profession could advance an idea so \textcolor{blue}{patently wrong}, even as today I
  can \textcolor{blue}{scarcely believe} that some people \textcolor{blue}{resist} an idea so \textcolor{blue}{patently
  right}.  (Savage et al., 1962, p76)
\end{quote}
\end{frame}
\end{document}
\subsection{A stronger form of the WCP}
\begin{frame}
\frametitle{A stronger form of the WCP}
\begin{itemize}
\item We consider the concept of \textcolor{blue}{ancillarity}.
\item This has several different definitions in the Statistics literature; the one we use is close to that of Cox and Hinkley (1974, Section 2.2).
\end{itemize}
\begin{block}{Definition (Ancillarity)}
  \textcolor{red}{$Y$} is \textcolor{blue}{ancillary} in the experiment \textcolor{red}{$\mathcal{E} = \{ \mathcal{X}
  \times \mathcal{Y}, \Theta, f_{X,Y}(x, y \, | \,\theta)\}$} exactly when \textcolor{red}{$f_{X,Y}$} factorises as
\begin{eqnarray*}
\textcolor{red}{f_{X,Y}(x, y \, | \, \theta) \ = \ f_Y(y)f_{X | Y}(x \, | \, y, \theta)}.
\end{eqnarray*}
\end{block}
\begin{itemize}
\item The marginal distribution of \textcolor{red}{$Y$} is completely specified: it does not depend on \textcolor{red}{$\theta$}.
\item We could extend this to consider an extended parameter set, say \textcolor{red}{$(\lambda, \theta)$} where \textcolor{red}{$\lambda$} is a \textcolor{blue}{nuisance parameter} and \textcolor{red}{$\theta$} is the parameter of interest.
    \item \textcolor{blue}{Ancillarity} would be that \textcolor{red}{$f_{Y}$} doesn't depend on \textcolor{red}{$\theta$} but may on \textcolor{red}{$\lambda$} whilst \textcolor{red}{$f_{X | Y}$} depends on \textcolor{red}{$\theta$ }but doesn't depend on \textcolor{red}{$\lambda$}.
\end{itemize}
\end{frame}
\begin{frame}
\begin{itemize}
\item Not all families of distributions will factorise in this way, but when they do, there are new possibilities for inference, based around stronger forms of the WCP.
\item A familiar example is that of a \textcolor{blue}{random sample size}: in a sample \textcolor{red}{$x = (x_{1}, \ldots, x_{n})$}, \textcolor{red}{$n$} may be the outcome of a random variable \textcolor{red}{$N$}.
\item  We seldom concern ourselves with the distribution of $N$ when we evaluate $x$; instead we treat \textcolor{red}{$N$} as \textcolor{blue}{known}. 
\item Equivalently, we treat \textcolor{red}{$N$} as \textcolor{blue}{ancillary} and \textcolor{blue}{condition} on \textcolor{red}{$N = n$}. 
\item In this case, we might think that inferences drawn from observing \textcolor{red}{$(n, x)$} should be the \textcolor{blue}{same} as those for \textcolor{red}{$x$} \textcolor{blue}{conditioned} on \textcolor{red}{$N = n$}.
\end{itemize}
\end{frame}
\begin{frame}
\begin{itemize}
\item When $Y$ is ancillary, we can consider the \textcolor{blue}{conditional experiment} 
\begin{eqnarray*}\label{eq:ancillary}
  \textcolor{red}{\mathcal{E}^{X \, | \,  y} = \{\mathcal{X}, \Theta, f_{X \, | \, Y}(x \, | \, y, \theta)\}}.
\end{eqnarray*}
\item That is, we treat $Y$ as known, and treat $X$ (conditional on $Y = y$) as the only random variable. 
\end{itemize}
\begin{block}{Principle 9: Strong Conditionality Principle, SCP}
  If \textcolor{red}{$Y$} is \textcolor{blue}{ancillary} in \textcolor{red}{$\mathcal{E}$}, then \textcolor{red}{$\Ev \left( \mathcal{E}, (x, y)
  \right) = \Ev( \mathcal{E}^{X | y}, x)$}.
\end{block}
\begin{itemize}
\item The SCP is invoked (implicitly) when we perform a \textcolor{blue}{regression} of $Y$ on $X$: $(X, Y)$ is random, but $X$ is treated as ancillary for the parameters in $f_{Y | X}$. We model $Y$ conditionally on $X$, treating $X$ as known.
\item Clearly \textcolor{blue}{the SCP implies the WCP}, with the experiment indicator $I \in \{1, 2\}$ being ancillary, since $p$ is known.
\end{itemize}
\end{frame}
\begin{frame}
\begin{block}{Theorem}
  $\mbox{SLP} \rightarrow \mbox{SCP}.$
\end{block}
\begin{block}{Proof} Suppose that $Y$ is ancillary in $\mathcal{E} =
  \{ \mathcal{X} \times \mathcal{Y}, \Theta, f_{X,Y}(x, y \, | \, \theta)\}$.  Thus, for all $\theta \in \Theta$, 
\begin{eqnarray*}
  f_{X,Y}(x, y \, | \, \theta) &  = & \textcolor{red}{f_Y(y)}f_{X | Y}(x \, | \, y, \theta) \\
&   =  & \textcolor{red}{c(y)} f_{X | Y}(x \, | \, y, \theta)
\end{eqnarray*}
Then the \textcolor{blue}{SLP} implies that
\begin{eqnarray*}
  \textcolor{red}{\Ev \left( \mathcal{E}, (x, y) \right) \ = \ \Ev( \mathcal{E}^{X | y}, x)} ,
\end{eqnarray*}
  as required. \hfill $\Box$
  \end{block}
  \begin{itemize}
  \item From Birnbaum's Theorem, $(\mbox{WIP} \wedge \mbox{WCP}) \leftrightarrow \mbox{SLP}$ so, as $\mbox{SLP} \rightarrow \mbox{SCP}$, the WIP allows us to `upgrade' the WCP to the SCP.
  \end{itemize}
  \end{frame}
\subsection{The Likelihood Principle in practice}
\begin{frame}
\frametitle{The Likelihood Principle in practice}
\begin{itemize}
\item We consider whether there is any inferential approach which respects the SLP?  Or do all inferential approaches respect it?  
\end{itemize}
A \textcolor{blue}{Bayesian statistical model} is the collection 
\begin{eqnarray*}
\textcolor{red}{\mathcal{E}_{B} \ = \ \{\mathcal{X}, \Theta, f_{X}(x \, | \, \theta), \pi(\theta)\}}.
\end{eqnarray*}
The \textcolor{blue}{posterior distribution} is \textcolor{red}{$\pi(\theta \, | \, x) = c(x)f_{X}(x \, | \, \theta)\pi(\theta)$} where $c(x)$ is the normalising constant,
\begin{eqnarray*}
c(x) & = & \left\{\int_{\Theta}  f_{X}(x \, | \, \theta)\pi(\theta) \, d\theta\right\}^{-1}.
\end{eqnarray*}
\begin{itemize}
\item All knowledge about $\theta$ given the data $x$ are represented by \textcolor{red}{$\pi(\theta \, | \, x)$}.
\item \textcolor{blue}{Any} inferences made about $\theta$ are derived from this distribution.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Consider two Bayesian models with the \textcolor{blue}{same} prior distribution, \textcolor{red}{$\mathcal{E}_{B, 1} = \{\mathcal{X}_{1}, \Theta, f_{X_{1}}(x_{1} \, | \, \theta), \pi(\theta)\}$} and \textcolor{red}{$\mathcal{E}_{B, 2} = \{\mathcal{X}_{2}, \Theta, f_{X_{2}}(x_{2} \, | \, \theta), \pi(\theta)\}$}
\item Suppose that \textcolor{red}{$f_{X_{1}}(x_{1} \, | \, \theta) = c(x_{1}, x_{2})f_{X_{2}}(x_{2} \, | \, \theta)$}. Then 
\begin{eqnarray*}
\pi_{1}(\theta \, | \, x_{1}) \ = \ c(x_{1})\textcolor{red}{f_{X_{1}}(x_{1} \, | \, \theta)}\pi(\theta) 
& = & c(x_{1}) \textcolor{red}{c(x_{1}, x_{2})f_{X_{2}}(x_{2} \, | \, \theta)} \pi(\theta) \\
& = &  \pi_{2}(\theta \, | \, x_{2}) \label{eq:postsame}
\end{eqnarray*}
\item Hence, the posterior distributions are the \textcolor{blue}{same}. Consequently, the \textcolor{blue}{same inferences} are drawn from either model and so \textcolor{blue}{the Bayesian approach satisfies the SLP}.
\item This assumes that $\pi(\theta)$ does not depend upon the form of the data.
\item Some methods for making \textcolor{blue}{default} choices for $\pi(\theta)$ depend on $f_X(x \, | \, \theta)$, notably Jeffreys priors and reference priors. These methods \textcolor{blue}{violate the SLP}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Maximum likelihood estimation clearly \textcolor{blue}{satisfies the SLP} and methods, such as penalised likelihood theory, have been generated to satisfy the SLP.
\item However, inference tools used in the classical approach typically \textcolor{blue}{violate the SLP}.
\item Inference techniques depend upon the \textcolor{blue}{sampling distribution} and so they depend on the \textcolor{blue}{whole sample space} \textcolor{red}{$\mathcal{X}$} and not just the \textcolor{blue}{observed} \textcolor{red}{$x \in \mathcal{X}$}. 
\item Sampling distribution depends on values of $f_{X}$ other than $L(\theta; x) = f_{X}(x \, | \, \theta)$.
\item For a statistic $T(X)$, $MSE(T \, | \, \theta) = Var(T \, | \, \theta) + \mbox{bias}(T \, | \, \theta)^{2}$ depends upon the first and second moments of the distribution of $T \, | \, \theta$.
\end{itemize}
\end{frame}
\begin{frame}
\begin{block}{Example, Robert (2007)}
\begin{itemize}
\item Suppose that $X_{1}, X_{2}$ are iid $N(\theta, 1)$ so that 
\begin{eqnarray*}
f(x_{1}, x_{2} \, | \, \theta) \ \propto \ \textcolor{red}{\exp\left\{-(\overline{x} - \theta)^{2}\right\}}. 
\end{eqnarray*}
\item Consider the alternate model for the \textcolor{blue}{same} parameter $\theta$
\begin{eqnarray*}
g(x_{1}, x_{2} \, | \, \theta) \ = \ \pi^{-\frac{3}{2}}\frac{\textcolor{red}{\exp\left\{-(\overline{x} - \theta)^{2}\right\}}}{1 + (x_{1} - x_{2})^{2}}
\end{eqnarray*}
\item Thus, \textcolor{red}{$f(x_{1}, x_{2} \, | \, \theta) \propto g(x_{1}, x_{2} \, | \, \theta)$} as a function of $\theta$.  If the \textcolor{blue}{SLP} is applied, then inference about $\theta$ should be the \textcolor{red}{same in both models}.
\item The distribution of $g$ is quite \textcolor{blue}{different} from that of $f$ and so estimators of $\theta$ will have different classical properties if they do not depend only on \textcolor{red}{$\overline{x}$}.
\item For example, $g$ has heavier tails than $f$ and so respective confidence intervals may differ between the two.
\end{itemize}
\end{block}
\end{frame}
\begin{frame}
\begin{itemize}
\item Suppose that $\Ev(\mathcal{E}, x)$ \textcolor{blue}{depends} on the value of \textcolor{red}{$f_{X}(x' \, | \, \theta)$} for some \textcolor{red}{$x' \neq x$}. Then, typically, $\Ev$ \textcolor{blue}{does not respect} the SLP.
\item We could create an alternate experiment \textcolor{red}{$\mathcal{E}_{1} = \{\mathcal{X}, \Theta, f_{1}(x \, | \, \theta)\}$} where:
\begin{itemize} 
\item \textcolor{red}{$f_{1}(x \, | \, \theta)  = f_{X}(x \, | \, \theta)$} for the \textcolor{blue}{observed} \textcolor{red}{$x$}.
\item \textcolor{red}{$f_{1}(x \, | \, \theta)  \neq f_{X}(x \, | \, \theta)$} for \textcolor{blue}{all} \textcolor{red}{$x \in \mathcal{X}$}.
\end{itemize}
\item In particular, that \textcolor{red}{$f_{1}(x' \, | \, \theta)  \neq f_{X}(x'\, | \, \theta)$}.
\begin{itemize}
\item Let \textcolor{red}{$\tilde{x} \neq x, x'$} and set 
\begin{eqnarray*}
f_{1}(x' \, | \, \theta) & = & \alpha f_{X} (x' \, | \, \theta) + \beta f_{X} (\tilde{x} \, | \, \theta) \\
f_{1}(\tilde{x} \, | \, \theta) & = & (1-\alpha) f_{X} (x' \, | \, \theta) + (1-\beta) f_{X} (\tilde{x} \, | \, \theta)
\end{eqnarray*}
\item By suitable choice of $\alpha$, $\beta$ we can redistribute the mass to ensure \textcolor{red}{$f_{1}(x' \, | \, \theta)  \neq f_{X}(x'\, | \, \theta)$}. We then let \textcolor{red}{$f_{1} = f_{X}$} elsewhere.
\end{itemize}
\item Consequently, whilst \textcolor{red}{$f_{1}(x \, | \, \theta)  = f_{X}(x \, | \, \theta)$} we will not have that \textcolor{red}{$\Ev(\mathcal{E}, x) = \Ev(\mathcal{E}_{1}, x)$} and so will violate the SLP.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
The two main difficulties with violating the SLP are:
\begin{enumerate}
\item To reject the SLP is to reject at least one of the WIP and the WCP. Yet both of these principles seem self-evident. Therefore violating the SLP is either illogical or obtuse.
\item In their everyday practice, statisticians use the SRP (ignoring the intentions of the experimenter) which is not self-evident, but is implied by the SLP. If the SLP is violated, it needs an alternative justification which has not yet been forthcoming.
\end{enumerate}
\end{frame}
\subsection{Reflections}
\begin{frame}
\frametitle{Reflections}
\begin{itemize}
\item This chapter does not explain how to choose $\Ev$ but instead describes desirable properties of $\Ev$.
\item What is evaluated is the algorithm, the method by which $(\mathcal{E}, x)$ is turned into an inference about the parameter $\theta$.
\item It is quite possible that statisticians of quite different persuasions will produce \textcolor{blue}{effectively identical} inferences from \textcolor{blue}{different} algorithms.  
\item A Bayesian statistician might produce a 95\% High Density Region, and a classical statistician a 95\% confidence set, but they might be effectively the same set. 
\item Primary concern for the auditor is why the particular inference method was chosen and they might also ask if the statistician is worried about the SLP.
\item Classical statistician might argue a long-run frequency property but the client might wonder about \textcolor{blue}{their} interval.
\end{itemize}
\end{frame}
\section{Statistical Decision Theory}
\subsection{Introduction}
\begin{frame}
  \frametitle{Introduction}
\begin{itemize}
\item \textcolor{blue}{Statistical Decision Theory} allows us to consider ways to construct the \textcolor{red}{$\Ev$} function that reflects our needs, which will vary from application to application, and which assesses the consequences of making a \textcolor{blue}{good or bad} inference.
\item The set of possible inferences, or \textcolor{blue}{decisions}, is termed the \textcolor{blue}{decision space}, denoted \textcolor{red}{$\mathcal{D}$}. 
\item For each \textcolor{red}{$d \in \mathcal{D}$}, we want a way to assess the consequence of how good or bad the \textcolor{blue}{choice} of decision \textcolor{red}{$d$} was under the \textcolor{blue}{event} \textcolor{red}{$\theta$}.
\end{itemize}
\begin{block}{Definition (Loss function)}
A loss function is any function $L$ from $\Theta \times \mathcal{D}$ to $[0, \infty)$.
\end{block}
\begin{itemize}
\item The loss function measures the \textcolor{blue}{penalty} or error, \textcolor{red}{$L(\theta, d)$} of the \textcolor{blue}{decision} \textcolor{red}{$d$} when the \textcolor{blue}{parameter} takes the value \textcolor{red}{$\theta$}. 
\item Thus, larger values indicate worse consequences. 
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\}
The three main types of inference about $\theta$ are 
\begin{enumerate}
\item point estimation, 
\item set estimation,
\item hypothesis testing.
\end{enumerate}
  It is a great conceptual and practical simplification that Statistical Decision Theory \textcolor{blue}{distinguishes} between these three types simply according to their \textcolor{blue}{decision spaces}.
  
 \ \newline 
\renewcommand{\arraystretch}{1.2}
\bigskip\noindent% 
\begin{tabular}{lp{2.75in}}
\hline
  Type of inference & Decision space $\mathcal{D}$ \\
\hline
  Point estimation & The parameter space, $\Theta$. \\
  Set estimation & A set of subsets of $\Theta$. \\
  Hypothesis testing & A specified partition of $\Theta$, denoted $\mathcal{H}$. \\
\hline\hline
\end{tabular}
\end{frame}
\subsection{Bayesian statistical decision theory}
\begin{frame}
\frametitle{Bayesian statistical decision theory}
In a Bayesian approach, a \textcolor{blue}{statistical decision problem} \textcolor{red}{$[\Theta, \mathcal{D}, \pi(\theta), L(\theta, d)]$} has the following ingredients.
\begin{enumerate} 
\item The possible values of the parameter: \textcolor{red}{$\Theta$}, the \textcolor{blue}{parameter space}.
    \item The set of possible decisions: \textcolor{red}{$\mathcal{D}$}, the \textcolor{blue}{decision space}.
\item The \textcolor{blue}{probability distribution} on $\Theta$, \textcolor{red}{$\pi(\theta)$}. For example,
\begin{enumerate}
\item this could be a \textcolor{blue}{prior} distribution, \textcolor{red}{$\pi(\theta) = f(\theta)$}.
\item this could be a \textcolor{blue}{posterior} distribution, \textcolor{red}{$\pi(\theta) = f(\theta \, | \, x)$} following the receipt of some \textcolor{blue}{data} \textcolor{red}{$x$}.
    \item this could be a \textcolor{blue}{posterior} distribution \textcolor{red}{$\pi(\theta) = f(\theta \, | \, x, y)$} following the receipt of some \textcolor{blue}{data} \textcolor{red}{$x$,$y$}.
\end{enumerate}
    \item The \textcolor{blue}{loss function} \textcolor{red}{$L(\theta, d)$}.
\end{enumerate}
In this setting, \textcolor{blue}{only} \textcolor{red}{$\theta$} is \textcolor{blue}{random} and we can calculate the \textcolor{blue}{expected loss}, or \textcolor{blue}{risk}. 
\end{frame}
\begin{frame}
%\frametitle{\ } 
\begin{block}{Definition (Risk)} 
The \textcolor{blue}{risk} of decision \textcolor{red}{$d \in \mathcal{D}$} under the distribution \textcolor{red}{$\pi(\theta)$} is 
\begin{eqnarray*}
\rho(\pi(\theta), d) & = & \int_{\theta} L(\theta, d)\pi(\theta) \, d\theta. \label{eq4.2}
\end{eqnarray*}
\end{block}
We choose $d$ to \textcolor{blue}{minimise} this risk.
\begin{block}{Definition (Bayes rule and Bayes risk)}\
The \textcolor{blue}{Bayes risk} \textcolor{red}{$\rho^{*}(\pi)$} minimises the expected loss,
\begin{eqnarray*}
 \rho^{*}(\pi) & = & \inf_{d \in \mathcal{D}} \rho(\pi, d)
\end{eqnarray*}
with respect to $\pi(\theta)$. A decision \textcolor{red}{$d^{*} \in \mathcal{D}$} for which \textcolor{red}{$\rho(\pi, d^{*}) = \rho^{*}(\pi)$} is a \textcolor{blue}{Bayes rule} against $\pi(\theta)$.
\end{block}
The Bayes rule may not be unique, and in weird cases it might not exist. We \textcolor{blue}{solve} \textcolor{red}{$[\Theta, \mathcal{D}, \pi(\theta), L(\theta, d)]$} by \textcolor{blue}{finding} \textcolor{red}{$\rho^{*}(\pi)$} and (at least one) \textcolor{red}{$d^{*}$}.
\end{frame} 
\begin{frame}
%\frametitle{\ }
\begin{block}{Example - quadratic loss}
Suppose that $\Theta \subset \mathbb{R}$ and we wish to find a \textcolor{blue}{point estimate} for $\theta$. We consider the loss function $\textcolor{red}{L(\theta, d) \ = \ (\theta - d)^{2}}$.
\begin{itemize}
\item The \textcolor{blue}{risk} of decision $d$ is 
\begin{eqnarray*}
\rho(\pi, d) \ = \ \mathbb{E}\{L(\theta, d) \, | \, \theta \sim \pi(\theta)\} & = & \mathbb{E}_{(\pi)}\{(\theta - d)^{2}\} \\
& = &  \mathbb{E}_{(\pi)}(\theta^{2}) - 2d\mathbb{E}_{(\pi)}(\theta) + d^{2},
\end{eqnarray*}
where \textcolor{red}{$\mathbb{E}_{(\pi)}(\cdot)$} denotes the expectation with respect to \textcolor{red}{$\pi(\theta)$}. 
\item Differentiating with respect to $d$ we have
\begin{eqnarray*}
\frac{\partial}{\partial d} \rho(\pi, d) & = & -2\mathbb{E}_{(\pi)}(\theta) + 2d.
\end{eqnarray*}
\item So, the \textcolor{blue}{Bayes rule} is \textcolor{red}{ $d^{*} = \mathbb{E}_{(\pi)}(\theta)$}. 
\end{itemize}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Example - quadratic loss (continued)}
\begin{itemize}
\item The corresponding \textcolor{blue}{Bayes risk} is
\begin{eqnarray*}
\textcolor{red}{\rho^{*}(\pi)} \ = \ \rho(\pi, d^{*}) & = & \mathbb{E}_{(\pi)}(\theta^{2}) - 2d^{*}\mathbb{E}_{(\pi)}(\theta) + (d^{*})^{2} \\
& = & Var_{(\pi)}(\theta) + (d^{*} - \mathbb{E}_{(\pi)}(\theta))^{2} \\
& = & \textcolor{red}{Var_{(\pi)}(\theta)}
\end{eqnarray*}
where $Var_{(\pi)}(\theta)$ is the variance of $\theta$ computed with respect to $\pi(\theta)$.
\end{itemize}
\begin{enumerate}
\item If \textcolor{red}{$\pi(\theta) = f(\theta)$}, a \textcolor{blue}{prior} for $\theta$, then the \textcolor{blue}{Bayes rule} of an \textcolor{blue}{immediate decision} is \textcolor{red}{$d^{*} = \mathbb{E}(\theta)$} with corresponding \textcolor{blue}{Bayes risk} \textcolor{red}{$\rho^{*} = Var(\theta)$}.
\item If we observe \textcolor{blue}{sample data} $x$ then the \textcolor{blue}{Bayes rule} given this \textcolor{blue}{sample information} is \textcolor{red}{$d^{*} = \mathbb{E}(\theta \, | \, X)$} with corresponding \textcolor{blue}{Bayes risk} \textcolor{red}{$\rho^{*} = Var(\theta \, | \, X)$ }as \textcolor{red}{$\pi(\theta) = f(\theta \, | \, x)$}.
\end{enumerate} 
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Typically we solve:
\begin{enumerate}
\item $[\Theta, \mathcal{D}, \textcolor{red}{f(\theta)}, L(\theta, d)]$, the \textcolor{blue}{immediate decision} problem,
\item $[\Theta, \mathcal{D}, \textcolor{red}{f(\theta \, | \, x)}, L(\theta, d)]$, the decision problem \textcolor{blue}{after sample information}. 
\end{enumerate}
\item We may also want to consider the \textcolor{blue}{risk of the sampling procedure}, before observing the sample, to decide whether or not to sample. 
\item We now consider both \textcolor{red}{$\theta$} and \textcolor{red}{$X$} as \textcolor{blue}{random}.
\item For each \textcolor{blue}{possible sample}, we need to specify which decision to make. 
\end{itemize}
\begin{block}{Definition (Decision rule)}
A decision rule \textcolor{red}{$\delta(x)$} is a function from $\mathcal{X}$ into $\mathcal{D}$,
\begin{eqnarray*}
\delta : \mathcal{X} \rightarrow \mathcal{D}.
\end{eqnarray*}  
If $X = x$ is the observed value of the sample information then $\delta(x)$ is the decision that \textcolor{blue}{will be taken}. The collection of all decision rules is denoted by $\Delta$ so that $\delta \in \Delta \Rightarrow \delta(x) \in \mathcal{D} \ \forall x \in X$. 
\end{block}
\end{frame} 
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item We wish to solve the problem $[\Theta, \Delta, \textcolor{red}{f(\theta, x), L(\theta, \delta(x))}]$. 
\end{itemize}
\begin{block}{Definition (Bayes (decision) rule and risk of the sampling procedure)}
The decision rule \textcolor{red}{$\delta^{*}$} is a \textcolor{blue}{Bayes (decision) rule} exactly when
\begin{eqnarray*}
\mathbb{E}\{L(\theta, \delta^{*}(X))\} & \leq & \mathbb{E}\{L(\theta, \delta(X))\} \label{eq:BRT}
\end{eqnarray*}
for all $\delta(x) \in \mathcal{D}$. The corresponding risk \textcolor{red}{$\rho^{*} = \mathbb{E}\{L(\theta, \delta^{*}(X))\}$} is termed the \textcolor{blue}{risk of the sampling procedure}.
\end{block}
\begin{itemize}
\item If the sample information consists of $X = (X_{1}, \ldots, X_{n})$ then $\rho^{*}$ will be a function of $n$ and so can be used to help determine \textcolor{blue}{sample size choice}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Bayes rule theorem, BRT}
Suppose that a Bayes rule exists for $[\Theta, \mathcal{D}$, $f(\theta \, | \, x), L(\theta, d)]$. Then 
\begin{eqnarray*}
 \delta^*(x) = \argmin_{d \in \mathcal{D}} \mathbb{E}(L(\theta, d) \, | \, X = x).\label{eq:delta*}
\end{eqnarray*}
\end{block}
\begin{block}{Proof} Let $\delta$ be arbitrary. Then
\begin{eqnarray*}
\mathbb{E}\{L(\theta, \delta(X))\} & = & \int_{x}\int_{\theta} L(\theta, \delta(x))\textcolor{red}{f(\theta, x)}\, d\theta dx \nonumber \\
& = & \int_{x}\int_{\theta} L(\theta, \delta(x))\textcolor{red}{f(\theta \, | \, x)f(x)} \, d\theta dx \nonumber \\
& = & \int_{x} \left\{\textcolor{blue}{\int_{\theta} L(\theta, \delta(x))f(\theta \, | \, x) \, d\theta} \right\} f(x) \, dx  \nonumber \\
& = & \int_{x} \textcolor{blue}{\mathbb{E}\{L(\theta, \delta(x)) \, | \, X\}} f(x) \, dx  
\end{eqnarray*}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
Now, as \textcolor{red}{$f(x) > 0$}, the $\delta^{*} \in \Delta$ which minimises $\mathbb{E}\{L(\theta, \delta(X))\}$ may equivalently be found as the $\delta^{*}$ which satisfies
\begin{eqnarray*}
\rho(f(\theta), \delta^{*}) & = & \inf_{\delta(x) \in \mathcal{D}} \mathbb{E}\{L(\theta, \delta(x)) \, | \, X\}, \label{eq4.4}
\end{eqnarray*}
giving the result. \hfill $\Box$
\end{block}
\begin{itemize}
\item The minimisation of expected loss over the space of \textcolor{blue}{all} functions from $\mathcal{X}$ to $\mathcal{D}$ can be achieved by the \textcolor{blue}{pointwise minimisation} over $\mathcal{D}$ of the expected loss \textcolor{blue}{conditional} on $X = x$.  
\item The risk of the sampling procedure is \textcolor{red}{$\rho^{*} = \mathbb{E}[\mathbb{E}\{L(\theta, \delta^{*}(x)) \, | \, X\}]$}.
\end{itemize}
\begin{block}{Example - quadratic loss}
We have \textcolor{red}{$\delta^{*} = \mathbb{E}(\theta \, | \, X)$} and \textcolor{red}{$\rho^{*} = \mathbb{E}\{Var(\theta \, | \, X)\}$}.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
We could consider $\Delta$, the \textcolor{blue}{set of decision rules}, to be our possible \textcolor{blue}{set of inferences} about $\theta$ when the sample is observed so that \textcolor{red}{$\Ev(\mathcal{E}, x)$} is \textcolor{red}{$\delta^*(x)$}. We thus have the following result.
\begin{block}{Theorem} 
The Bayes rule for the posterior decision respects the strong likelihood principle.
\end{block}
\begin{block}{Proof}
If we have two Bayesian models with the \textcolor{blue}{same} prior distribution then if \textcolor{red}{$f_{X_{1}}(x_{1} \, | \, \theta) = c(x_{1}, x_{2})f_{X_{2}}(x_{2} \, | \, \theta)$} the corresponding posterior distributions are the \textcolor{blue}{same} and so the corresponding Bayes rule (and risk) is the same. \hfill $\Box$
\end{block}
\end{frame}
\subsection{Admissible rules}
\begin{frame}
\frametitle{Admissible rules}
\begin{itemize}
\item Bayes rules rely upon a \textcolor{blue}{prior distribution} for \textcolor{red}{$\theta$}: the risk is a function of \textcolor{red}{$d$} \textcolor{blue}{only}.
\item In \textcolor{blue}{classical statistics}, there is \textcolor{blue}{no distribution} for $\theta$ and so another approach is needed. 
\begin{block} {Definition (The classical risk)}
For a decision rule $\delta (x)$, the classical risk for the model $\mathcal{E} = \{\mathcal{X}, \Theta, f_{X}(x \, | \, \theta)\}$ is
\begin{eqnarray*}
R(\theta, \delta) & = & \int_{X} L(\theta, \delta(x))f_{X}(x \, | \, \theta) \, dx.
\end{eqnarray*}
\end{block} 
\item The classical risk is thus, for each \textcolor{red}{$\delta$}, a \textcolor{blue}{function} of \textcolor{red}{$\theta$}. 
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Example}
Let $X = (X_{1}, \ldots, X_{n})$ where $X_{i} \sim N(\theta, \sigma^{2})$ and $\sigma^{2}$ is known. Suppose that $L(\theta, d) = (\theta - d)^{2}$ and consider a conjugate prior $\theta \sim N(\mu_{0}, \sigma^{2}_{0})$. Possible decision functions include:
\begin{enumerate}
\item \textcolor{red}{$\delta_{1}(x) = \overline{x}$}, the \textcolor{blue}{sample mean}.
\item \textcolor{red}{$\delta_{2}(x) = \mbox{med}\{x_{1}, \ldots, x_{n}\} = \tilde{x}$}, the \textcolor{blue}{sample median}.
\item \textcolor{red}{$\delta_{3}(x) = \mu_{0}$}, the \textcolor{blue}{prior mean}.
\item \textcolor{red}{$\delta_{4}(x) = \mu_{n}$}, the \textcolor{blue}{posterior mean} where
\begin{eqnarray*}
\mu_{n} & = & \left(\frac{1}{\sigma_{0}^{2}} + \frac{n}{\sigma^{2}}\right)^{-1}\left(\frac{\textcolor{red}{\mu_{0}}}{\sigma_{0}^{2}} + \frac{n\textcolor{red}{\overline{x}}}{\sigma^{2}}\right),
\end{eqnarray*}
the weighted average of the prior and sample mean accorded to their respective precisions.
\end{enumerate}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Example - continued}
The respective classical risks are
\begin{enumerate}
\item \textcolor{red}{$R(\theta,\delta_{1}) = \frac{\sigma^{2}}{n}$}, a \textcolor{blue}{constant} for $\theta$, since $\overline{X} \sim N(\theta, \sigma^{2}/n)$.
\item \textcolor{red}{$R(\theta,\delta_{2}) = \frac{\pi \sigma^{2}}{2n}$}, a \textcolor{blue}{constant} for $\theta$, since $\tilde{X} \sim N(\theta, \pi\sigma^{2}/2n)$ (approximately).
\item \textcolor{red}{$R(\theta, \delta_{3}) = (\theta - \mu_{0})^{2} = \sigma^{2}_{0}\left(\frac{\theta - \mu_{0}}{\sigma_{0}}\right)^{2}$}.
\item \textcolor{red}{$R(\theta, \delta_{4}) = \left(\frac{1}{\sigma_{0}^{2}} + \frac{n}{\sigma^{2}}\right)^{-2}\left\{\frac{1}{\sigma_{0}^{2}}\left(\frac{\theta - \mu_{0}}{\sigma_{0}}\right)^{2} + \frac{n}{\sigma^{2}}\right\}$}.
\end{enumerate}
Which decision do we choose? We observe that \textcolor{red}{$R(\theta,\delta_{1}) < R(\theta,\delta_{2})$} for \textcolor{blue}{all} \textcolor{red}{$\theta \in \Theta$} but other comparisons depend upon $\theta$.
\end{block}
\begin{itemize}
\item The accepted approach for classical statisticians is to narrow the set of possible
decision rules by \textcolor{blue}{ruling out} those that are obviously \textcolor{blue}{bad}. 
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Definition (Admissible decision rule)}
A decision rule \textcolor{red}{$\delta_{0}$} is \textcolor{blue}{inadmissible} if there exists a decision rule \textcolor{red}{$\delta_{1}$} which \textcolor{blue}{dominates} it, that is 
\begin{eqnarray*}
\textcolor{red}{R(\theta, \delta_{1}) \ \leq \ R(\theta, \delta_{0})}
\end{eqnarray*}
\textcolor{blue}{for all} \textcolor{red}{$\theta \in \Theta$} with \textcolor{red}{$ R(\theta, \delta_{1}) < R(\theta, \delta_{0})$} for \textcolor{blue}{at least one} value \textcolor{red}{$\theta_{0} \in \Theta$}. If no such $\delta_{1}$ exists then \textcolor{red}{$\delta_{0}$} is \textcolor{blue}{admissible}.
\end{block}
\begin{itemize}
\item If \textcolor{red}{$\delta_{0}$} is \textcolor{blue}{dominated} by \textcolor{red}{$\delta_{1}$} then the classical risk of \textcolor{red}{$\delta_{0}$} is \textcolor{blue}{never smaller} than that of \textcolor{red}{$\delta_{1}$} and \textcolor{red}{$\delta_{1}$} has a \textcolor{blue}{smaller} risk for \textcolor{red}{$\theta_{0}$}. 
\item Thus, you would \textcolor{blue}{never} want to use \textcolor{red}{$\delta_{0}$}.\footnote{Here I am assuming that all other considerations are the same in the two cases: e.g.\ for all $x \in \mathcal{X}$, $\delta_{1}(x)$ and $\delta_{0}(x)$ take about the same amount of resource to compute.}
\item The accepted approach is to \textcolor{blue}{reduce} the set of possible decision rules under consideration by only \textcolor{blue}{using admissible rules}.
\end{itemize}
\end{frame}
\begin{frame} 
%\frametitle{\ }
\begin{itemize}
\item We now show that \textcolor{blue}{admissible rules} can be related to a \textcolor{blue}{Bayes rule} \textcolor{red}{$\delta^{*}$} for a \textcolor{blue}{prior distribution} \textcolor{red}{$\pi(\theta)$}.
\end{itemize}
\begin{block}{Theorem}
If a prior distribution $\pi(\theta)$ is strictly positive for all $\Theta$ with finite Bayes risk and the classical risk, $R(\theta, \delta)$, is a continuous function of $\theta$ for all $\delta$, then the \textcolor{blue}{Bayes rule} \textcolor{red}{$\delta^{*}$} is \textcolor{blue}{admissible}.
\end{block}
\begin{block}{Proof (Robert, 2007)}
Letting \textcolor{red}{$f(\theta, x) = f_{X}(x \, | \, \theta)\pi(\theta)$} we have 
\begin{eqnarray*}
\mathbb{E}\{L(\theta, \delta(X))\} & = & \int_{x}\int_{\theta} L(\theta, \delta(x))\textcolor{red}{f(\theta, x)}\, d\theta dx \nonumber \\
& = & \int_{\theta} \left\{\textcolor{blue}{\int_{x} L(\theta, \delta(x))f_{X}(x \, | \, \theta) \, dx} \right\} \pi(\theta)\, d\theta  \nonumber \\
& = & \int_{\theta} \textcolor{blue}{R(\theta, \delta)} \pi(\theta) \, d\theta  
\end{eqnarray*}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
\begin{itemize}
\item Suppose that the \textcolor{blue}{Bayes rule} \textcolor{red}{$\delta^{*}$} is \textcolor{blue}{inadmissible} and \textcolor{blue}{dominated} by \textcolor{red}{$\delta_{1}$}.
\item Thus, in an open set $C$ of $\theta$, \textcolor{red}{$R(\theta, \delta_{1}) < R(\theta, \delta^{*})$ }with \textcolor{red}{$R(\theta, \delta_{1}) \leq R(\theta, \delta^{*})$} elsewhere. 
\item Consequently, \textcolor{red}{$\mathbb{E}\{L(\theta, \delta_{1}(X))\} < \mathbb{E}\{L(\theta, \delta^{*}(X))\}$} which is a \textcolor{blue}{contradiction} to \textcolor{blue}{$\delta^{*}$} being the \textcolor{blue}{Bayes rule}. \hfill $\Box$
\end{itemize}
\end{block}
\begin{itemize}
\item The relationship between a Bayes rule with prior $\pi(\theta)$ and an admissible decision rule is even stronger.
\item The following result was derived by \href{https://en.wikipedia.org/wiki/Abraham_Wald}{\textcolor{light-blue}{\underline{Abraham Wald (1902-1950)}}}
\end{itemize}
\begin{block}{Wald's Complete Class Theorem, CCT}
 In the case where the parameter space $\Theta$  and sample space $\mathcal{X}$ are finite, a decision rule \textcolor{red}{$\delta$} is \textcolor{blue}{admissible if and only if it is a Bayes rule} for some \textcolor{blue}{prior} distribution \textcolor{red}{$\pi(\theta)$} with strictly positive values.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item An illuminating blackboard proof of this result can be found in \textcolor{blue}{Cox and Hinkley (1974, Section 11.6)}.
\item There are \textcolor{blue}{generalisations} of this theorem to non-finite decision sets, parameter spaces, and sample spaces but the results are \textcolor{blue}{highly technical}. 
\item We'll proceed \textcolor{blue}{assuming} the more general result, which is that \textcolor{blue}{a decision rule is admissible if and only if it is a Bayes rule for some prior distribution} $\pi(\theta)$, which holds for practical purposes.
\end{itemize}
So what does the CCT say? 
\begin{enumerate}
\item \textcolor{blue}{Admissible decision rules respect the SLP}. This follows from the fact that admissible rules are Bayes rules which respect the SLP. This provides support for using admissible decision rules.
\item If you select a \textcolor{blue}{Bayes rule} according to some positive prior distribution $\pi(\theta)$ then you \textcolor{blue}{cannot} ever choose an \textcolor{blue}{inadmissible} decision rule. 
\end{enumerate}
\end{frame}
\subsection{Point estimation}
\begin{frame}
\frametitle{Point estimation}
\begin{itemize}
\item We now look at possible choices of loss functions for different types of inference.
\item For \textcolor{blue}{point estimation} the decision space is \textcolor{red}{$\mathcal{D} = \Theta$}, and the loss function \textcolor{red}{$L(\theta, d)$} represents the (negative) consequence of choosing \textcolor{red}{$d$} as a \textcolor{blue}{point estimate} of \textcolor{red}{$\theta$}. 
\item It will not be often that an obvious loss function \textcolor{red}{${L: \Theta \times \Theta \to \mathbb{R}}$} presents itself. There is a need for a \textcolor{blue}{generic} loss function which is acceptable over a \textcolor{blue}{wide range} of applications. 
\end{itemize}
Suppose that \textcolor{red}{$\Theta$} is a \textcolor{blue}{convex subset} of \textcolor{red}{$\mathbb{R}^{p}$}. A natural choice is a \textcolor{blue}{convex loss function},
\begin{eqnarray*}\label{eq:convexL}
  \textcolor{red}{L(\theta, d) \ = \ h(d - \theta)}
\end{eqnarray*}
where \textcolor{red}{$h : \mathbb{R}^{p} \to \mathbb{R}$} is a smooth non-negative convex function with \textcolor{red}{$h(0) = 0$}.  
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item This type of loss function asserts that small errors are much more tolerable than large ones. 
\item One possible further restriction is that \textcolor{red}{$h$} is an \textcolor{blue}{even function}, \textcolor{red}{$h(d - \theta) = h(\theta - d)$}.
\item In this case, \textcolor{red}{$L(\theta, \theta + \epsilon) = L(\theta, \theta - \epsilon)$} so that \textcolor{blue}{under-estimation} incurs the \textcolor{blue}{same} loss as \textcolor{blue}{over-estimation}. 
\item We saw previously, that for \textcolor{blue}{quadratic loss} \textcolor{red}{$\Theta \subset \mathbb{R}$}, \textcolor{red}{$L(\theta, d) = (\theta - d)^{2}$}, the Bayes rule was the \textcolor{blue}{expectation} of \textcolor{red}{$\pi(\theta)$}. As we will see, this attractive feature can be extended to more dimensions. 
\item There are many situations where this is \textcolor{blue}{not} appropriate and the loss function should be asymmetric and a generic loss function should be replaced by a more specific one.
\end{itemize} 
\end{frame} 
\begin{frame}
The \textcolor{blue}{bilinear loss function} for \textcolor{red}{$\Theta \subset \mathbb{R}$} is, for $\alpha, \beta > 0$, 
\textcolor{red}{\begin{eqnarray*}
L(\theta, d) & = & \left\{\begin{array}{ll}
\alpha(\theta - d) & \mbox{if $d \leq \theta$}, \\
\beta(d - \theta) & \mbox{if $d \geq \theta$}.
\end{array} \right.
\end{eqnarray*}}
\begin{itemize}
\item The Bayes rule is a \textcolor{red}{$\frac{\alpha}{\alpha + \beta}$}\textcolor{blue}{-fractile} of $\pi(\theta)$.
\item If $\alpha = \beta = 1$ then \textcolor{red}{$L(\theta, d) = |\theta - d|$}, the \textcolor{blue}{absolute loss} which gives a Bayes rule of the \textcolor{blue}{median} of \textcolor{red}{$\pi(\theta)$}.
\item $|\theta - d|$ is smaller that $(\theta - d)^{2}$ for $|\theta - d| > 1$ and so absolute loss is smaller than quadratic loss for large deviations. Thus, it takes less account of the tails of $\pi(\theta)$ leading to the choice of the median.
\item If $\alpha > \beta$, so $\frac{\alpha}{\alpha + \beta} >0.5$, then under-estimation is penalised more than over-estimation and so that Bayes rule is more likely to be an over-estimate. 
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Example}
If $\Theta \in \mathbb{R}^{p}$, the Bayes rule $\delta^{*}$ associated with the distribution $\pi(\theta)$ and the quadratic loss
\begin{eqnarray*}
L(\theta, d) & = & (d - \theta)^{T} Q \, (d - \theta)
\end{eqnarray*}
is the \textcolor{blue}{expectation} \textcolor{red}{$\mathbb{E}_{(\pi)}(\theta)$} for \textcolor{blue}{every} positive-definite symmetric $p \times p$ matrix $Q$.
\end{block}
\begin{block}{Example (Robert, 2007), $Q = \Sigma^{-1}$} 
Suppose $X \sim N_{p}(\theta, \Sigma)$ where the known variance matrix $\Sigma$ is diagonal with elements $\sigma^{2}_{i}$ for each $i$. Then $\mathcal{D} = \mathbb{R}^{p}$. A possible loss function is
\begin{eqnarray*}
L(\theta, d) & = &\sum_{i=1}^{p} \left(\frac{d_{i} - \theta_{i}}{\sigma_{i}}\right)^{2}
\end{eqnarray*}
so that the total loss is the sum of the squared component-wise errors.
\end{block}
\end{frame} 
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item As the \textcolor{blue}{Bayes rule} for \textcolor{red}{$L(\theta, d)  =  (d - \theta)^{T} Q \, (d - \theta)$} \textcolor{blue}{does not depend} upon \textcolor{red}{$Q$}, it is the \textcolor{blue}{same} for an uncountably large class of loss functions. 
\item If we apply the \textcolor{blue}{Complete Class Theorem} to this result we see that for quadratic loss, a \textcolor{blue}{point estimator} for $\theta$ is \textcolor{blue}{admissible} if and only if it is the \textcolor{blue}{conditional expectation} with respect to some positive prior distribution $\pi(\theta)$. 
\item The value, and interpretability, of the quadratic loss can be further observed by noting that, from a \textcolor{blue}{Taylor series expansion}, an even, differentiable and strictly convex loss function can be \textcolor{blue}{approximated} by a quadratic loss function. 
\end{itemize}
\end{frame}
\subsection{Set estimation}
\begin{frame}
\frametitle{Set estimation}
\begin{itemize}
\item For set estimation the \textcolor{blue}{decision space} is a \textcolor{blue}{set of subsets} of \textcolor{red}{$\Theta$} so that each \textcolor{red}{$d \subset \Theta$}. 
\item There are two contradictory requirements for set estimators of $\Theta$.  
\begin{enumerate}
\item We want the sets to be small.
\item We also want them to contain $\theta$.
\end{enumerate}
\item A simple way to represent these two requirements is to consider the loss function 
\begin{eqnarray*}
 \textcolor{red}{L(\theta, d) \ = \ |d| + \kappa(1 - \mathbbm{1}_{\theta \in d})} \label{eq:CL2}
\end{eqnarray*}
for some \textcolor{red}{$\kappa > 0$} where \textcolor{red}{$|d|$} is the \textcolor{blue}{volume} of \textcolor{red}{$d$}. 
\item The value of \textcolor{red}{$\kappa$} controls the \textcolor{blue}{trade-off} between the two requirements. 
\begin{itemize}
\item If \textcolor{red}{$\kappa \downarrow 0$} then minimising the expected loss will always produce the \textcolor{blue}{empty set}.  
\item If \textcolor{red}{$\kappa \uparrow \infty$} then minimising the expected loss will always produce \textcolor{red}{$\Theta$}.  
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item For loss functions of the form \textcolor{red}{$L(\theta, d) \ = \ |d| + \kappa(1 - \mathbbm{1}_{\theta \in d})$} we'll show there is a a simple necessary condition for a rule to be a Bayes rule. 
\end{itemize}
\begin{block}{Definition (Level set)}
A set \textcolor{red}{$d \subset \Theta$} is a \textcolor{blue}{level set} of the posterior distribution exactly when \textcolor{red}{$d = \{\theta \, : \, \pi(\theta \, | \, x) \geq k\}$} for some $k$.
\end{block}
\begin{block}{Theorem (Level set property, LSP)}
If \textcolor{red}{$\delta^{*}$} is a \textcolor{blue}{Bayes rule} for \textcolor{red}{$L(\theta, d) \ = \ |d| + \kappa(1 - \mathbbm{1}_{\theta \in d})$} then it is a \textcolor{blue}{level set} of the posterior distribution.
\end{block}
\begin{block}{Proof}
Note that
\begin{eqnarray*}
\mathbb{E}\{L(\theta, d) \, | \, X\} & = & |d| + \kappa(1- \mathbb{E}(\mathbbm{1}_{\theta \in d} \, | \, X)) \\
& = & |d| + \kappa\mathbb{P}(\theta \notin d \, | \, X). \label{eq:LSPone} 
\end{eqnarray*}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
\begin{itemize}
\item For fixed $x$, we show that if $d$ is \textcolor{blue}{not} a level set of the posterior distribution then there is a \textcolor{red}{$d' \neq d$} which has a \textcolor{blue}{smaller} expected loss so that \textcolor{red}{$\delta^{*}(x) \neq d$}. 
\item Suppose that \textcolor{red}{$d$} is \textcolor{blue}{not a level set} of \textcolor{red}{$\pi(\theta \, | \, x)$}. Then there is a \textcolor{red}{$\theta \in d$} and \textcolor{red}{$\theta' \notin d$} for which \textcolor{red}{$\pi(\theta' \, | \, x) > \pi(\theta \, | \, x)$}. 
\item Let \textcolor{red}{$d' = d \cup d\theta' \setminus d\theta$} where \textcolor{red}{$d\theta$} is the tiny region of $\Theta$ around \textcolor{red}{$\theta$} and \textcolor{red}{$d\theta'$} is the tiny region of $\Theta$ around \textcolor{red}{$\theta'$} for which \textcolor{red}{$|d\theta| = |d\theta'|$}. 
\item Then \textcolor{red}{$|d'| = |d|$} but
\begin{eqnarray*}
 \textcolor{red}{\mathbb{P}(\theta \notin d' \, | \, X) <  \mathbb{P}(\theta \notin d \, | \, X)}
\end{eqnarray*}
Thus, \textcolor{red}{$\mathbb{E}\{L(\theta, d') \, | \, X\} < \mathbb{E}\{L(\theta, d) \, | \, X\}$} showing that \textcolor{red}{$\delta^{*}(x) \neq d$}. \hfill $\Box$
\end{itemize}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item The \textcolor{blue}{Level Set Property Theorem} states that \textcolor{red}{$\delta$} having the  level set property is \textcolor{blue}{necessary} for \textcolor{red}{$\delta$} to be a \textcolor{blue}{Bayes rule} for loss functions of the form \textcolor{red}{$L(\theta, d) \ = \ |d| + \kappa(1 - \mathbbm{1}_{\theta \in d})$}.
\item The \textcolor{blue}{Complete Class Theorem} states that being a \textcolor{blue}{Bayes rule} is a \textcolor{blue}{necessary} condition for \textcolor{red}{$\delta$} to be \textcolor{blue}{admissible}. 
\item Being a \textcolor{blue}{level set of a posterior} distribution for \textcolor{blue}{some prior} distribution \textcolor{red}{$\pi(\theta)$} is a \textcolor{blue}{necessary} condition for being \textcolor{blue}{admissible} for loss functions of this form.
\item \textcolor{blue}{Bayesian HPD regions} satisfy the necessary condition for being a set estimator.
\item \textcolor{blue}{Classical set estimators} achieve a similar outcome if they are \textcolor{blue}{level sets of the likelihood function}, because the posterior is proportional to the likelihood under a uniform prior distribution.\footnote{In the case where $\Theta$ is unbounded, this prior distribution may have to be truncated to be proper.}
\end{itemize}
\end{frame}
\subsection{Hypothesis tests}
\begin{frame}
\frametitle{Hypothesis tests}
\begin{itemize}
\item For hypothesis tests, the decision space is a \textcolor{blue}{partition} of $\Theta$, denoted
\begin{eqnarray*}
  \textcolor{red}{\mathcal{H} \ := \ \{ H_0, H_1, \dots, H_d\}} .
\end{eqnarray*}
\item Each element of \textcolor{red}{$\mathcal{H}$} is termed a \textcolor{blue}{hypothesis}.
\item The loss function $L(\theta, H_i)$ represents the (negative) consequences of choosing element \textcolor{red}{$H_i$}, when the true value of the parameter is $\theta$. 
\item  It would be usual for the loss function to satisfy 
\begin{eqnarray*}
 \textcolor{red}{\theta \in H_i \implies L(\theta, H_i) \ = \ \min_{j} L(\theta, H_{j})}
\end{eqnarray*}
on the grounds that an \textcolor{blue}{incorrect} choice of element \textcolor{blue}{should never} incur a \textcolor{blue}{smaller} loss than the \textcolor{blue}{correct} choice.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Consider the test of \textcolor{red}{$H_{0}: \theta \in \Theta_{0}$ versus $H_{1}: \theta \in \Theta_{1}$} where $\Theta_{1} = \Theta \setminus \Theta_{0}$. Let $\mathcal{D} = \{d_{0}, d_{1}\}$ where $d_{i}$ corresponds to accepting $H_{i}$. A generic loss function is the \textcolor{blue}{0-1 ('zero-one') loss} function
\begin{eqnarray*}
\textcolor{red}{L(\theta, d_{i})} & \textcolor{red}{=} & \textcolor{red}{\left\{\begin{array}{ll}
0 & \mbox{if $\theta \in \Theta_{i}$},\\
1 & \mbox{if $\theta \notin \Theta_{i}$}.
\end{array} \right.}
\end{eqnarray*}
\item The \textcolor{blue}{classical risk} is the probability of making a wrong decision,
\begin{eqnarray*}
R(\theta, \delta) & = & \left\{\begin{array}{ll}
\mathbb{P}(\delta(X) = d_{1} \, | \, \theta) & \mbox{if $\theta \in \Theta_{0}$}, \\
\mathbb{P}(\delta(X) = d_{0} \, | \, \theta) & \mbox{if $\theta \in \Theta_{1}$},
\end{array} \right.
\end{eqnarray*}
which correspond to the familiar Type I and Type II errors.
\item The Bayes rule is to choose $H_{0}$ if \textcolor{red}{$\mathbb{P}_{\pi}(\theta \in \Theta_{0}) > \mathbb{P}_{\pi}(\theta \in \Theta_{1})$} and $H_{1}$ otherwise, where $\mathbb{P}_{\pi}(\cdot)$ is the probability when $\theta \sim \pi(\theta)$.
\item Hence, if \textcolor{red}{$\pi(\theta) = f(\theta \, | \, x)$}, the \textcolor{blue}{Bayes rule} is to choose the hypothesis with the \textcolor{blue}{largest posterior probability}. 
\end{itemize}
\end{frame}
\begin{frame}
\begin{itemize}
\item This approach can be naturally extended to multiple hypotheses $\textcolor{red}{\mathcal{H} = \{ H_0, H_1, \dots, H_d\}}$ which partition $\Theta$ by taking 
\begin{eqnarray*}
 \textcolor{red}{ L(\theta, H_i) \ = \ 1 - \mathbbm{1}_{\{\theta \in H_i\}}}.
\end{eqnarray*}
i.e., \textcolor{blue}{zero} if \textcolor{red}{$\theta \in H_i$}, and one if it is not. 
\item For the posterior decision, the \textcolor{blue}{Bayes rule} is to select the hypothesis with the \textcolor{blue}{largest posterior probability}. 
\item However, this loss function is hard to defend as being realistic.
\item If we choose $H_{i}$ and it turns out that $\theta \notin H_{i}$ then the inference is wrong and the loss is the same irrespective of where $\theta$ lies.
\item An alternative approach is to co-opt the theory of \textcolor{blue}{set estimators}.
\item The statistician can use her set estimator \textcolor{red}{$\delta$} to make at least some distinctions between the members of \textcolor{red}{$\mathcal{H}$}:
\begin{itemize}
\item \textcolor{blue}{Accept} \textcolor{red}{$H_i$} exactly when \textcolor{red}{$\delta(x) \subset H_i$},
\item \textcolor{blue}{Reject} \textcolor{red}{$H_i$} exactly when \textcolor{red}{$\delta(x) \cap H_i = \emptyset$},
\item \textcolor{blue}{Undecided} about \textcolor{red}{$H_i$} otherwise.
\end{itemize}
\end{itemize}
\end{frame}
\section{Confidence sets and p-values}
\subsection{Confidence procedures and confidence sets}
\begin{frame}
\frametitle{Confidence procedures and confidence sets}
\begin{itemize}
\item We consider \textcolor{blue}{interval estimation}, or more generally \textcolor{blue}{set estimation}. 
\item Under the model \textcolor{red}{$\mathcal{E} = \{\mathcal{X}, \Theta, f_{X}(x \, | \, \theta)\}$}, for given data \textcolor{red}{$X = x$}, we wish to construct a set \textcolor{red}{$C = C(x) \subset \Theta$} and the \textcolor{blue}{inference} is the statement that \textcolor{red}{$\theta \in C$}. 
\item If \textcolor{red}{$\theta \in \mathbb{R}$} then the set estimate is typically an \textcolor{blue}{interval}.
\end{itemize}
 \begin{block}{Definition (Confidence procedure)}
 A \textcolor{blue}{random set} \textcolor{red}{$C(X)$} is a level-\omalpha \textcolor{blue}{confidence procedure} exactly when 
\begin{eqnarray*}
\textcolor{red}{\mathbb{P}(\theta \in C(X) \, | \, \theta) \ \geq \ 1 - \alpha}
\end{eqnarray*}
\textcolor{blue}{for all} \textcolor{red}{$\theta \in \Theta$}. $C$ is an \textcolor{blue}{exact} level-\omalpha confidence procedure if the  probability \textcolor{blue}{equals} \textcolor{red}{\omalpha} for all $\theta$.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item The value \textcolor{red}{$\mathbb{P}(\theta \in C(X) \, | \, \theta)$} is termed the \textcolor{blue}{coverage} of \textcolor{red}{$C$} at \textcolor{red}{$\theta$}. 
\item Exact is a special case: typically \textcolor{red}{$\mathbb{P}(\theta \in C(X) \, | \, \theta)$} will depend upon \textcolor{red}{$\theta$}. 
\item The procedure is thus \textcolor{blue}{conservative}: for a \textcolor{blue}{given} \textcolor{red}{$\theta_{0}$} the \textcolor{blue}{coverage} may be much \textcolor{blue}{higher} than \omalpha.
\end{itemize}
\begin{block}{Uniform example}
\begin{itemize}
\item Let $X_{1}, \ldots, X_{n}$ be independent and identically distributed \textcolor{red}{$\mbox{Unif}(0, \theta)$} random variables where $\theta > 0$. Let \textcolor{red}{$Y = \max\{X_{1}, \ldots, X_{n}\}$}. 
\item  We consider two possible sets: \textcolor{red}{$(aY, bY)$} where $1 \leq a < b$ and \textcolor{red}{$(Y + c, Y + d)$} where $0 \leq c < d$. 
\begin{enumerate}
\item \textcolor{red}{$\mathbb{P}(\theta \in (aY, bY) \, | \, \theta) =  \left(\frac{1}{a}\right)^{n} - \left(\frac{1}{b}\right)^{n}$}. Thus, the coverage probability of the interval \textcolor{blue}{does not depend} upon \textcolor{red}{$\theta$}. 
\item \textcolor{red}{$\mathbb{P}(\theta \in (Y + c, Y + d) \, | \, \theta) = \left(1 - \frac{c}{\theta}\right)^{n} - \left(1 - \frac{d}{\theta}\right)^{n}$}. In this case, the coverage probability of the interval \textcolor{blue}{does depend} upon \textcolor{red}{$\theta$}.
\end{enumerate}
\end{itemize}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item We distinguish between the confidence \textcolor{blue}{procedure} $C$, which is a \textcolor{blue}{random interval} and so a function for each possible $x$, and the result when $C$ is \textcolor{blue}{evaluated} at the \textcolor{blue}{observation} \textcolor{red}{$x$}, which is a \textcolor{blue}{set} in $\Theta$. 
\end{itemize}
\begin{block}{Definition (Confidence set)}
The observed $C(x)$ is a level-\omalpha confidence set exactly when the random $C(X)$ is a level-\omalpha confidence procedure.  
\end{block}
\begin{itemize}
\item If \textcolor{red}{$\Theta \subset \mathbb{R}$} and \textcolor{red}{$C(x)$} is \textcolor{blue}{convex}, i.e.\ an interval, then a confidence set (interval) is represented by a lower and upper value. 
\item The \textcolor{blue}{challenge} with confidence procedures is to construct one with a \textcolor{blue}{specified level}: to do this we \textcolor{blue}{start with the level} and then construct a $C$ guaranteed to have this level. 
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Definition (Family of confidence procedures)}
\begin{itemize}
\item \textcolor{red}{$C(X; \alpha)$} is a \textcolor{blue}{family} of confidence procedures exactly when \textcolor{red}{$C(X; \alpha)$} is a level-$(1-\alpha)$ confidence procedure for \textcolor{blue}{every} \textcolor{red}{$\alpha \in [0, 1]$}. 
\item \textcolor{red}{$C$} is a \textcolor{blue}{nesting family} exactly when \textcolor{red}{$\alpha < \alpha'$} implies that \textcolor{red}{$C(x; \alpha') \subset C(x; \alpha)$}.
\end{itemize}
\end{block}
\begin{itemize}
\item If we start with a family of confidence procedures for a specified model, then we can compute a confidence set for any level we choose.
\end{itemize}
\end{frame}
\subsection{Constructing confidence procedures}
\begin{frame}
\frametitle{Constructing confidence procedures}
\begin{itemize}
\item The general approach to construct a confidence procedure is to \textcolor{blue}{invert a test statistic}. 
\item In the Uniform example, the coverage of the procedure \textcolor{red}{$(aY, bY)$} does not depend upon $\theta$ because the coverage probability could be expressed in terms of \textcolor{red}{$T = Y/\theta$} where the distribution of \textcolor{red}{$T$} did \textcolor{blue}{not depend} upon \textcolor{red}{$\theta$}.
\begin{itemize}
\item  $T$ is an example of a \textcolor{blue}{pivot} and confidence procedures are straightforward to compute from a pivot.
\item However, a drawback to this approach in general is that there is \textcolor{blue}{no hard and fast method} for finding a pivot.
\end{itemize}
\item An alternate method which does work generally is to exploit the property that \textcolor{blue}{\textit{every}} \textcolor{blue}{confidence procedure} corresponds to a \textcolor{blue}{hypothesis test} and vice versa. 
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
Consider a hypothesis test where we have to decide either to \textcolor{blue}{accept} that an hypothesis \textcolor{red}{$H_{0}$} is true \textcolor{blue}{or} to \textcolor{blue}{reject} \textcolor{red}{$H_{0}$} in favour of an alternative hypothesis \textcolor{red}{$H_{1}$} based on a sample $x \in \mathcal{X}$. 
\begin{itemize}
\item The set of $x$ for which $H_{0}$ is rejected is called the \textcolor{blue}{rejection region}.
\item The complement, where $H_{0}$ is accepted, is the \textcolor{blue}{acceptance region}. 
\item A hypothesis test can be constructed from \textcolor{blue}{any statistic} $T = T(X)$.
\end{itemize}
\begin{block}{Definition (Likelihood Ratio Test, LRT)}
 The likelihood ratio test (LRT) statistic for testing \textcolor{red}{$H_{0}: \theta \in \Theta_{0}$} versus \textcolor{red}{$H_{1}: \theta \in \Theta_{0}^{\rm c}$}, where \textcolor{red}{$\Theta_{0} \cup \Theta_{0}^{\rm c} = \Theta$}, is
  \begin{eqnarray*}
   \textcolor{red}{ \lambda(x) \ = \ \frac{\sup_{\theta \in \Theta_{0}}L_{X}(\theta; x)}{\sup_{\theta \in \Theta} L_{X}(\theta; x)}}. \label{eq:LRT}
  \end{eqnarray*}
  A LRT at significance level $\alpha$ has a \textcolor{blue}{rejection region} of the form \textcolor{red}{$\{x \,  : \, \lambda(x) \leq c\}$} where $0 \leq c \leq 1$ is chosen so that \textcolor{red}{$\mathbb{P}(\mbox{Reject $H_{0}$} \, | \, \theta) \leq \alpha$} for all \textcolor{red}{$\theta \in \Theta_{0}$}.
  \end{block}
  \end{frame}
  \begin{frame}
  %\frametitle{\ }
  \begin{block}{Example}
  \begin{itemize}
 \item Let $X = (X_{1}, \ldots, X_{n})$ and suppose that the $X_{i}$ are independent and identically distributed \textcolor{red}{$N(\theta, \sigma^{2})$} random variables where \textcolor{red}{$\sigma^{2}$} is \textcolor{blue}{known}.
 \item Consider the \textcolor{blue}{likelihood ratio test} for \textcolor{red}{$H_{0}: \theta = \theta_{0}$} versus \textcolor{red}{$H_{1}: \theta \neq \theta_{0}$}. Then, as the \textcolor{blue}{maximum likelihood estimate (mle)} of \textcolor{red}{$\theta$} is \textcolor{red}{$\overline{x}$},
  \begin{eqnarray*}
      \textcolor{red}{\lambda(x)} \ = \ \frac{L_{X}(\theta_{0}; x)}{L_{X}(\overline{x}; x)} & = & \exp\left\{-\frac{1}{2\sigma^{2}} \sum_{i=1}^{n} \left( (x_{i} - \theta_{0})^{2} -  (x_{i} - \overline{x})^{2}\right)\right\} \\
                 & = & \textcolor{red}{\exp\left\{-\frac{1}{2\sigma^{2}} n(\overline{x} - \theta_{0})^{2}\right\}}.
    \end{eqnarray*}
    Notice that, under \textcolor{red}{$H_{0}$}, \textcolor{red}{$\frac{\sqrt{n}(\overline{X} - \theta_{0})}{\sigma} \sim N(0, 1)$} so that
    \begin{eqnarray*}
     \textcolor{red}{ -2\log \lambda(X) \ = \ \frac{n(\overline{X} - \theta_{0})^{2}}{\sigma^{2}} \sim \chi^{2}_{1}}, \label{eq:motivateLRT}
    \end{eqnarray*}
    the chi-squared distribution with one degree of freedom. 
 \end{itemize}
 \end{block}
 \end{frame}
 \begin{frame}
 %\frametitle{\ }
 \begin{block}{Example continued}
 \begin{itemize}
 \item The \textcolor{blue}{rejection region} is \textcolor{red}{$\{x \,  : \, \lambda(x) \leq c\} = \{x \,  : \, -2 \log \lambda(x) \geq k\}$}.
 \item Setting \textcolor{red}{$k = \chi^{2}_{1, \alpha}$}, where \textcolor{red}{$\mathbb{P}(\chi^{2}_{1} \geq \chi^{2}_{1, \alpha}) = \alpha$}, gives a test at the \textcolor{blue}{exact} significance level \textcolor{red}{$\alpha$}. 
 \end{itemize}
 The \textcolor{blue}{acceptance region} of this test is \textcolor{red}{$\{x \,  : \, -2 \log \lambda(x) < \chi^{2}_{1, \alpha}\}$} where
    \begin{eqnarray*}
      \mathbb{P}\left(\left.\frac{n(\overline{X} - \theta_{0})^{2}}{\sigma^{2}} < \chi^{2}_{1, \alpha} \, \right| \, \theta = \theta_{0} \right) & = & 1- \alpha. \label{eq:accep}
    \end{eqnarray*}
    This holds \textcolor{blue}{for all} \textcolor{red}{$\theta_{0}$} and so, additionally rearranging,
    \begin{eqnarray*}
      \mathbb{P}\left(\left. \overline{X} - \sqrt{\chi^{2}_{1, \alpha}} \frac{\sigma}{\sqrt{n}} < \theta < \overline{X} + \sqrt{\chi^{2}_{1, \alpha}} \frac{\sigma}{\sqrt{n}}  \, \right| \, \theta \right) &  = & 1- \alpha.\label{eq:confy1}
    \end{eqnarray*} 
Thus, \textcolor{red}{$C(X) = (\overline{X} - \sqrt{\chi^{2}_{1, \alpha}} \frac{\sigma}{\sqrt{n}}, \overline{X} + \sqrt{\chi^{2}_{1, \alpha}} \frac{\sigma}{\sqrt{n}})$} is an \textcolor{blue}{exact} level-\textcolor{red}{$(1-\alpha)$} \textcolor{blue}{confidence procedure} with $C(x)$ the corresponding confidence set. 
 \end{block}
 \end{frame}
 \begin{frame}
 %\frametitle{\ }
 \begin{itemize}
\item Note that we obtained the level-$(1-\alpha)$ \textcolor{blue}{confidence procedure} by \textcolor{blue}{inverting} the \textcolor{blue}{acceptance region} of the level $\alpha$ \textcolor{blue}{significance test}. 
\item This correspondence, or duality, between acceptance regions of tests and confidence sets is a \textcolor{blue}{general property}.
\end{itemize}
 \begin{block}{Theorem (Duality of Acceptance Regions and Confidence Sets)}
\begin{enumerate}
\item For each \textcolor{red}{$\theta_{0} \in \Theta$}, let \textcolor{red}{$A(\theta_{0})$} be the \textcolor{blue}{acceptance region} of a test of \textcolor{red}{$H_{0}: \theta = \theta_{0}$} at significance level \textcolor{red}{$\alpha$}. For each \textcolor{red}{$x \in \mathcal{X}$}, define \textcolor{red}{$C(x) = \{\theta_{0} \, : \, x \in A(\theta_{0})\}$}. Then \textcolor{red}{$C(X)$} is a level-\textcolor{red}{$(1-\alpha)$} \textcolor{blue}{confidence procedure}.  
\item Let \textcolor{red}{$C(X)$} be a level-\textcolor{red}{$(1-\alpha)$} \textcolor{blue}{confidence procedure} and, for any \textcolor{red}{$\theta_{0} \in \Theta$}, define \textcolor{red}{$A(\theta_{0}) = \{ x \, : \, \theta_{0} \in C(x)\}$}. Then \textcolor{red}{$A(\theta_{0})$} is the \textcolor{blue}{acceptance region} of a test of \textcolor{red}{$H_{0}: \theta = \theta_{0}$} at \textcolor{blue}{significance level} \textcolor{red}{$\alpha$}.
\end{enumerate}
 \end{block}  
 \end{frame}
 \begin{frame}
 %\frametitle{\ }
 \begin{block}{Proof}
 \begin{enumerate}
 \item As we have a level \textcolor{red}{$\alpha$} \textcolor{blue}{test} for each \textcolor{red}{$\theta_{0} \in \Theta$} then \textcolor{red}{$\mathbb{P}( X \in A(\theta_{0})  \, | \, \theta = \theta_{0}) \geq 1- \alpha$}.  Since \textcolor{red}{$\theta_{0}$} is \textcolor{blue}{arbitrary} we may write $\theta$ instead of $\theta_{0}$ and so, for all \textcolor{red}{$\theta \in \Theta$},
 \begin{eqnarray*}
   \mathbb{P}(\theta \in C(X) \, | \, \theta) \ = \ \mathbb{P}(X \in A(\theta) \, | \, \theta) \ \geq 1 - \alpha.
 \end{eqnarray*}
 Hence, \textcolor{red}{$C(X)$} is a level-\textcolor{red}{$(1-\alpha)$} \textcolor{blue}{confidence procedure}. \newline
 \item For a test of \textcolor{red}{$H_{0}: \theta = \theta_{0}$}, the probability of a Type I error (rejecting $H_{0}$ when it is true) is
 \begin{eqnarray*}
   \mathbb{P}(X \notin A(\theta_{0}) \, | \, \theta = \theta_{0}) \ = \ \mathbb{P}(\theta_{0} \notin C(X) , | \, \theta = \theta_{0}) \ \leq \ \alpha
 \end{eqnarray*}
 since \textcolor{red}{$C(X)$} is a level-\textcolor{red}{$(1-\alpha)$} \textcolor{blue}{confidence procedure.} Hence, we have a \textcolor{blue}{test} at significance level \textcolor{red}{$\alpha$}. \hfill $\Box$
\end{enumerate} 
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
 A possibly easier way to understand the relationship between significance tests and confidence sets is by defining the set \textcolor{red}{$\{(x, \theta) \, : \, (x, \theta) \in \tilde{C}\}$} in the space \textcolor{red}{$\mathcal{X} \times \Theta$} where \textcolor{red}{$\tilde{C}$} is also a set in \textcolor{red}{$\mathcal{X} \times \Theta$}.
 \begin{itemize}
 \item For \textcolor{blue}{fixed} \textcolor{red}{$x$}, define the \textcolor{blue}{confidence set} as \textcolor{red}{$C(x) = \{\theta \, : \, (x, \theta) \in \tilde{C}\}$}.
 \item For \textcolor{blue}{fixed} \textcolor{red}{$\theta$}, define the \textcolor{blue}{acceptance region} as \textcolor{red}{$A(\theta) =  \{x \, : \, (x, \theta) \in \tilde{C}\}$}.
 \end{itemize}
 \begin{block}{Example revisited} Letting $x = (x_{1}, \ldots, x_{n})$, with \textcolor{blue}{$z^{2}_{\alpha/2} = \chi^{2}_{1, \alpha}$}, define the set
   \begin{eqnarray*}
     \{(x, \theta) \, : \, (x, \theta) \in \tilde{C}\} & = & \left\{(x, \theta) \, : \, -z_{\alpha/2}\sigma/\sqrt{n} < \overline{x} - \theta < z_{\alpha/2}\sigma/\sqrt{n}\right\}.
   \end{eqnarray*}
   The \textcolor{blue}{confidence set} is then $$C(x) = \left\{\textcolor{red}{\theta} \, : \, \overline{x} -z_{\alpha/2}\sigma/\sqrt{n}  < \textcolor{red}{\theta} < \overline{x}  + z_{\alpha/2}\sigma/\sqrt{n}\right\}$$ and \textcolor{blue}{acceptance region} $$A(\theta) = \left\{\textcolor{red}{x} \, : \, \theta -z_{\alpha/2}\sigma/\sqrt{n}  < \textcolor{red}{\overline{x}} < \theta + z_{\alpha/2}\sigma/\sqrt{n}\right\}.$$
\end{block}
\end{frame}
\subsection{Good choices of confidence procedures}
\begin{frame}
\frametitle{Good choices of confidence procedures}
\begin{itemize}
\item In the previous chapter, we showed that, under the generic loss \textcolor{red}{$L(\theta, d) \ = \ |d| + \kappa(1 - \mathbbm{1}_{\theta \in d})$}, a necessary condition for admissibility was that \textcolor{red}{$d$} was a \textcolor{blue}{level set} of the \textcolor{blue}{posterior} distribution.
\item We now proceed by consider confidence procedures that satisfy a \textcolor{blue}{level set} property for the \textcolor{blue}{likelihood} \textcolor{red}{$L_{X}(\theta; x) = f_{X}(x \, | \, \theta)$}.
\end{itemize}
\begin{block}{Definition (Level set property, LSP)}A confidence procedure $C$ has the level set property exactly when
\begin{eqnarray*}
C(x) & = & \{\theta \, : \, f_{X}(x \, | \, \theta) > g(x)\}
\end{eqnarray*}
for some $g : \mathcal{X} \rightarrow \mathbb{R}$.
\end{block}
We now show that we can construct a family of confidence procedures with the LSP. The result has pedagogic value, because it can be used to generate an uncountable number of families of confidence procedures, each with the level set property.
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Theorem}
Let \textcolor{red}{$h$} be any \textcolor{blue}{probability density function} for \textcolor{red}{$X$}.  Then 
\begin{eqnarray*}\label{eq:CPmarkov}
  \textcolor{red}{C_{h}(x; \alpha) \ := \  \{ \theta \in \Theta \, : \, f_{X}(x \, | \, \theta) > \alpha h(x)\}}
\end{eqnarray*}
is a family of confidence procedures, with the LSP.
\end{block}
\begin{block}{Proof}
First notice that if we let \textcolor{red}{$\mathcal{X}(\theta) := \{x \in \mathcal{X} \, : \, f_{X}(x \, |\,\theta) > 0\}$} then
\begin{eqnarray*}
  \textcolor{red}{\mathbb{E}(h(X) / f_{X}(X \, | \, \theta) \, | \, \theta)} & = & \int_{x \in \mathcal{X}(\theta)} \frac{ h(x) }{ f_{X}(x \, | \, \theta) }f_{X}(x \, | \, \theta) \, dx \nonumber\\
  & = & \int_{x \in \mathcal{X}(\theta)} h(x) \, \ \textcolor{red}{\leq \ 1} \label{eq:goonthen}
\end{eqnarray*}
because \textcolor{red}{$h$} is a \textcolor{blue}{probability density function}. 
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
Now,
\begin{eqnarray}
  \mathbb{P}(f_{X}(X \, | \,\theta) / h(X) \leq u \, | \, \theta) 
  & = & \mathbb{P}(h(X) / f_{X}(X \, | \, \theta) \geq 1 / u \, | \, \theta) \label{eq:markov1} \\
  & \leq & \frac{\textcolor{red}{\mathbb{E}( h(X)/ f_{X}(X \, | \, \theta) \, | \, \theta) }}{ 1 / u } \label{eq:markov2} \\
  & \leq & \frac{ \textcolor{red}{1} }{ 1 / u } = u \nonumber
\end{eqnarray}
where (\ref{eq:markov2}) follows from (\ref{eq:markov1}) by \textcolor{blue}{Markov's inequality}.\footnote{If $X$ is a nonnegative random variable and $a > 0$ then $\mathbb{P}(X \geq a) \leq \mathbb{E}(X)/a$.} \hfill $\Box$
\end{block}
\begin{itemize}
\item If we let \textcolor{red}{$g(x; \theta) = f_{X}(x \, | \,\theta) / h(x)$}, which may be infinite, then \textcolor{red}{$\mathbb{P}(g(X; \theta) \leq u \, | \, \theta) \leq u$}.
\item We will see later that this implies that \textcolor{red}{$g(x; \theta)$} is \textcolor{blue}{super-uniform}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Among the interesting choices for \textcolor{red}{$h$}, one possibility is \textcolor{red}{$h(x) =f_{X}(x \, | \, \theta_{0})$}, for some \textcolor{red}{$\theta_{0} \in \Theta$}. 
\item As \textcolor{red}{$f_{X}(x \, | \, \theta_{0}) > \alpha f_{X}(x \, | \, \theta_{0})$} we can \textcolor{blue}{construct} a level-\omalpha \textcolor{blue}{confidence procedure} whose confidence sets will \textcolor{blue}{always} contain \textcolor{red}{$\theta_{0}$}. 
\item This suggests an issue with confidence procedures: two statisticians may come to two \textcolor{blue}{different} conclusions about \textcolor{red}{$H_0 : \theta = \theta_0$} depending on the intervals \textcolor{blue}{they construct}. 
\item This illustrates why it is important to be able to \textcolor{blue}{account} for the \textcolor{blue}{choices} you make as a statistician.
\item The theorem utilises Markov's Inequality which is a \textcolor{blue}{very slack} result. It is likely that the \textcolor{blue}{coverage} of the corresponding family of confidence procedures will be \textcolor{blue}{much larger} than \textcolor{red}{\omalpha}.
\item A more desirable strategy would be to use an \textcolor{blue}{exact family} of confidence procedures which satisfy the \textcolor{blue}{LSP}, if one existed.
\end{itemize}
\end{frame}
\subsection{The linear model}
\begin{frame}
\frametitle{The linear model}
\begin{itemize}
\item We'll briefly discuss the \textcolor{blue}{linear model} and construct an \textcolor{blue}{exact family} of confidence procedures which satisfy the \textcolor{blue}{LSP}.
\item Let \textcolor{red}{$Y = (Y_{1}, \ldots, Y_{n})$} be an \textcolor{red}{$n$}-vector of observables with \textcolor{red}{$Y = X\theta + \epsilon$}.
\begin{itemize}
\item  \textcolor{red}{$X$} is an \textcolor{red}{$(n \times p)$} matrix\footnote{We typically use $X$ to denote a generic random variable and so it is not ideal to use it here for a specified matrix but this is the standard notation for linear models.} of \textcolor{blue}{regressors}, 
\item \textcolor{red}{$\theta$} is a \textcolor{red}{$p$}-vector of \textcolor{blue}{regression coefficients},
\item \textcolor{red}{$\epsilon$} is an \textcolor{red}{$n$}-vector of \textcolor{blue}{residuals}. 
\end{itemize}
\item Assume that \textcolor{red}{$\epsilon \sim N_{n}(0, \sigma^{2}I_{n})$}, the $n$-dimensional \textcolor{blue}{multivariate normal} distribution, where \textcolor{red}{$\sigma^{2}$} is \textcolor{blue}{known} and \textcolor{red}{$I_{n}$} is the $(n \times n)$ \textcolor{blue}{identity matrix}.
\item From properties of the multivariate normal distribution, it follows that \textcolor{red}{$Y \sim N_{n}(X\theta, \sigma^{2}I_{n})$}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
Now,
\begin{eqnarray*}
  L_{Y}(\theta; y) & = & \left(2\pi \sigma^{2}\right)^{-\frac{n}{2}} \exp\left\{-\frac{1}{2\sigma^{2}} (y - X\theta)^{T}(y-X\theta)\right\}. \label{eq:linear}
\end{eqnarray*}
Let \textcolor{red}{$\hat{\theta} = \hat{\theta}(y) = \left(X^{T}X\right)^{-1}X^{T}y$} then
\begin{eqnarray*}
  \textcolor{red}{(y - X\theta)^{T}(y-X\theta)} & = & (y-X\hat{\theta} + X\hat{\theta} - X\theta)^{T}(y-X\hat{\theta} + X\hat{\theta} - X\theta) \nonumber \\
                               & = & (y-X\hat{\theta})^{T}(y-X\hat{\theta}) + (X\hat{\theta} - X\theta)^{T}(X\hat{\theta} - X\theta) \nonumber \\
                               & = & \textcolor{red}{(y-X\hat{\theta})^{T}(y-X\hat{\theta}) + (\hat{\theta} - \theta)^{T}X^{T}X(\hat{\theta} - \theta)}. \label{eq:linearI}
\end{eqnarray*}
Thus, \textcolor{red}{$(y - X\theta)^{T}(y-X\theta)$} is \textcolor{blue}{minimised} when \textcolor{red}{$\theta = \hat{\theta}$} and so, \textcolor{red}{$\hat{\theta} = \left(X^{T}X\right)^{-1}X^{T}y$} is the \textcolor{blue}{mle} of \textcolor{red}{$\theta$}. The likelihood ratio is 
\begin{eqnarray*}
  \lambda(y) & = & \frac{L_{Y}(\theta; y)}{L_{Y}(\hat{\theta}; y)}\nonumber \\
             & = & \exp\left\{-\frac{1}{2\sigma^{2}}\left[ \textcolor{red}{(y - X\theta)^{T}(y-X\theta) - (y - X\hat{\theta})^{T}(y-X\hat{\theta})}\right]\right\} \nonumber\\
             & = & \exp\left\{-\frac{1}{2\sigma^{2}}\textcolor{red}{(\hat{\theta} - \theta)^{T}X^{T}X(\hat{\theta} - \theta)}\right\}\label{eq:linearII}
\end{eqnarray*}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Thus,
  \textcolor{red}{$-2\log \lambda(y)  = \frac{1}{\sigma^{2}}(\hat{\theta} - \theta)^{T}X^{T}X(\hat{\theta} - \theta)$}.
\item As \textcolor{red}{$\hat{\theta}(Y) = \left(X^{T}X\right)^{-1}X^{T}Y$} then, as \textcolor{red}{$Y \sim N_{n}(X\theta, \sigma^{2}I_{n})$}, $$\hat{\theta}(Y) \sim N_{p}\left(\theta, \sigma^{2}\left(X^{T}X\right)^{-1}\right)$$
\item Consequently, \textcolor{red}{$-2\log \lambda(Y) \sim \chi_{p}^{2}$}. \end{itemize}
Hence, with $\mathbb{P}(\chi^{2}_{p} \geq \chi^{2}_{p, \alpha}) = \alpha$,
\begin{eqnarray*}
  \textcolor{red}{C(y; \alpha)} & = & \left\{\theta \in \mathbb{R}^{p} \, : \, -2\log\lambda(y) \ = \ -2\log\frac{f_{Y}(y \, | \, \theta, \sigma^{2})}{f_{Y}(y \, | \, \hat{\theta}, \sigma^{2})} < \chi^{2}_{p, \alpha}\right\} \nonumber\\
               & = & \textcolor{red}{\left\{\theta \in \mathbb{R}^{p} \, : \, f_{Y}(y \, | \, \theta, \sigma^{2}) > \exp\left(-\frac{\chi^{2}_{p, \alpha}}{2}\right)f_{Y}(y \, | \, \hat{\theta}, \sigma^{2})\right\}} \label{eq:wilks}
\end{eqnarray*}
is a family of \textcolor{blue}{exact confidence procedures} for \textcolor{red}{$\theta$} which has the \textcolor{blue}{LSP}.
\end{frame}
\subsection{Wilks confidence procedures}
\begin{frame}
\frametitle{Wilks confidence procedures}
\begin{itemize}
\item This outcome, where we can find a family of exact confidence procedures with the LSP, is \textcolor{blue}{more-or-less unique} to the regression parameters of the \textcolor{blue}{linear model}.
\item  It is however found, \textcolor{blue}{approximately}, in the \textcolor{blue}{large} \textcolor{red}{$n$} behaviour of a much wider class of models. 
\end{itemize}
\begin{block}{Wilks' Theorem}
  Let $X = (X_{1}, \ldots, X_{n})$  where each $X_{i}$ is independent and identically distributed, \textcolor{red}{$X_{i} \sim f(x_{i} \, | \, \theta)$}, where $f$ is a \textcolor{blue}{regular model} and the \textcolor{blue}{parameter space} $\Theta$ is an open convex subset of \textcolor{red}{$\mathbb{R}^{p}$} (and invariant to $n$). The distribution of the statistic \textcolor{red}{$-2\log \lambda(X)$} converges to a \textcolor{blue}{chi-squared} distribution with \textcolor{red}{$p$} degrees of freedom as $n \rightarrow \infty$.
  \end{block}
  \begin{itemize}
  \item A working guideline to regular model is that $f$ must be smooth and differentiable in $\theta$; in particular, the support must not depend on $\theta$.  
  \end{itemize}
   \end{frame} 
  \begin{frame}
  %\frametitle{\ }
  \begin{itemize}
   \item The result dates back to Wilks (1938) and, as such, the resultant confidence procedures are often termed \textcolor{blue}{Wilks confidence procedures}.
  \item  Thus, if the conditions of Wilks' Theorem are met, 
   \begin{eqnarray*}
    \textcolor{red}{C(x; \alpha) \ = \ \left\{\theta \in \mathbb{R}^{p} \, : \, f_{X}(x \, | \, \theta) > \exp\left(-\frac{\chi^{2}_{p, \alpha}}{2}\right)f_{X}(x \, | \, \hat{\theta})\right\}} \label{eq:wilksI}
  \end{eqnarray*}
  is a family of \textcolor{blue}{approximately exact} confidence procedures which satisfy the LSP. 
  \item For a given model, the pertinent question is whether or not the approximation is a good one.
  \item We are thus interested in the \textcolor{blue}{level error}, the difference between the \textcolor{blue}{nominal level}, typically $(1 - \alpha)$ everywhere, and the \textcolor{blue}{actual level}, the actual minimum coverage everywhere,
  \begin{eqnarray*}
  \mbox{level error} & = &  \mbox{nominal level} - \mbox{actual level}.
\end{eqnarray*}
  \item Methods, such as \textcolor{blue}{bootstrap calibration}, described in DiCiccio and Efron (1996), exist which attempt to \textcolor{blue}{correct} for the level error.
  \end{itemize}
\end{frame}
\subsection{Significance procedures and duality}
\begin{frame}
\frametitle{Significance procedures and duality}
\begin{itemize}
\item  A \textcolor{blue}{hypothesis test} of \textcolor{red}{$H_{0}: \theta \in \Theta_{0}$} versus \textcolor{red}{$H_{1}: \theta \in \Theta_{0}^{\rm c}$}, where $\Theta_{0} \cup \Theta_{0}^{\rm c} = \Theta$, at significance level of $5\%$ (or any other specified value) returns \textcolor{blue}{one} bit of information, either we \textcolor{blue}{accept $H_{0}$} or \textcolor{blue}{reject $H_{0}$}. 
\item  We do not know whether the decision was \textcolor{blue}{borderline} or nearly \textcolor{blue}{conclusive}; i.e.\ whether, for rejection, $H_0$ and $C(x; 0.05)$ were close, or well-separated. 
\item Of more interest is to consider the \textcolor{blue}{smallest} value of \textcolor{red}{$\alpha$} for which \textcolor{red}{$C(x; \alpha)$} does \textcolor{blue}{not} intersect \textcolor{red}{$H_{0}$}. This value is termed the \textcolor{blue}{$p$-value}. 
\end{itemize}
\begin{block}{Definition ($p$-value)} 
A $p$-value \textcolor{red}{$p(X)$} is a \textcolor{blue}{statistic} satisfying \textcolor{red}{$p(x) \in [0,1]$} for every $x \in \mathcal{X}$. \textcolor{blue}{Small} values of \textcolor{red}{$p(x)$} support the hypothesis that \textcolor{red}{$H_{1}$} is true. A $p$-value is valid if, for every $\theta \in \Theta_{0}$ and every $\alpha \in [0, 1]$,
\begin{eqnarray*}
\mathbb{P}(p(X) \leq \alpha \, | \, \theta) & \leq & \alpha. \label{eq:pvalue}
\end{eqnarray*}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item If $p(X)$ is a valid $p$-value then a \textcolor{blue}{significance test} that rejects $H_{0}$ if and only if $p(X) \leq \alpha$ is a test with \textcolor{blue}{significance level} $\alpha$. 
\item In this part we introduce the idea of \textcolor{blue}{significance procedure} at level \textcolor{red}{$\alpha$}, deriving a \textcolor{blue}{duality} between it and a level \textcolor{red}{$1-\alpha$} \textcolor{blue}{confidence procedure}.  
\item Let $X$ and $Y$ be two \textcolor{blue}{scalar} random variables.  Then \textcolor{red}{$X$} \textcolor{blue}{stochastically dominates} \textcolor{red}{$Y$} exactly when \textcolor{red}{$\mathbb{P}(X \leq v) \leq \mathbb{P}(Y \leq v)$} for all \textcolor{red}{$v \in \mathbb{R}$}.
\item If \textcolor{red}{$U \sim \mbox{Unif}(0, 1)$} then \textcolor{red}{$\mathbb{P}(U \leq u)  = u$} for \textcolor{red}{$u \in [0, 1]$}. With this in mind, we make the following definition.
\end{itemize}
\begin{block}{Definition (Super-uniform)}
The random variable $X$ is \textcolor{blue}{super-uniform} exactly when it \textcolor{blue}{stochastically dominates} a standard \textcolor{blue}{uniform} random variable. That is
\begin{eqnarray*}
\mathbb{P}(X \leq u) & \leq & u \label{fn:sunif}
\end{eqnarray*}
for all $u \in [0, 1]$.
\end{block}
\begin{itemize}
\item Thus, for \textcolor{red}{$\theta \in \Theta_{0}$}, the \textcolor{blue}{$p$-value} \textcolor{red}{$p(X)$} is \textcolor{blue}{super-uniform}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item We now define a significance procedure. Note the similarities with the definitions of a confidence procedure which are not coincidental.
\end{itemize}
\begin{block}{Definition (Significance procedure)}
\begin{enumerate}
\item \textcolor{red}{$p : \mathcal{X} \rightarrow \mathbb{R}$} is a \textcolor{blue}{significance procedure} for \textcolor{red}{$\theta_{0} \in \Theta$} exactly when \textcolor{red}{$p(X)$} is \textcolor{blue}{super-uniform} under \textcolor{red}{$\theta_{0}$}. If \textcolor{red}{$p(X)$} is \textcolor{blue}{uniform} under $\theta_{0}$, then p is an \textcolor{blue}{exact} significance procedure for $\theta_{0}$. 
\item For \textcolor{red}{$X = x$}, \textcolor{red}{$p(x)$} is a \textcolor{blue}{significance level} or (observed) $p$-value for \textcolor{red}{$\theta_{0}$} exactly when \textcolor{red}{$p$} is a \textcolor{blue}{significance procedure} for \textcolor{red}{$\theta_{0}$}. 
\item \textcolor{red}{$p : \mathcal{X} \times \Theta \rightarrow \mathbb{R}$} is a \textcolor{blue}{family of significance procedures} exactly when \textcolor{red}{$p(x; \theta_{0})$} is a \textcolor{blue}{significance procedure} for \textcolor{red}{$\theta_{0}$} for \textcolor{blue}{every} \textcolor{red}{$\theta_{0} \in \Theta$}.
\end{enumerate}
\end{block}
\begin{itemize}
\item We now show that there is a duality between significance procedures and confidence procedures.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Duality Theorem}
\begin{enumerate}
\item Let \textcolor{red}{$p$} be a family of \textcolor{blue}{significance procedures}. Then
\begin{eqnarray*}
\textcolor{red}{C(x; \alpha) \ := \ \{\theta \in \Theta \, : \, p(x; \theta) > \alpha\}}
\end{eqnarray*}
is a nesting family of \textcolor{blue}{confidence procedures}.
\item Conversely, let \textcolor{red}{$C$} be a nesting family of \textcolor{blue}{confidence procedures}. Then
\begin{eqnarray*}
\textcolor{red}{p(x; \theta_{0}) \ := \ \inf \{\alpha \, : \, \theta_{0} \notin C(x; \alpha)\}}
\end{eqnarray*}
is a family of \textcolor{blue}{significance procedures}.
\end{enumerate}
If \textcolor{blue}{either} is \textcolor{blue}{exact}, then the \textcolor{blue}{other} is \textcolor{blue}{exact} as well.
\end{block} 
\begin{block}{Proof} \begin{itemize} \item If $p$ is a family of significance procedures then for any $\theta \in \Theta$,
$$\mathbb{P}(\theta \in C(X; \alpha) \, | \, \theta) \ = \ \mathbb{P}(p(X; \theta) > \alpha \, | \,\theta) \ = \ 1 - \mathbb{P}(p(X; \theta) \leq \alpha \, | \,\theta). $$ 
\end{itemize}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
\begin{itemize}
\item Now, as \textcolor{red}{$p$} is \textcolor{blue}{super-uniform} for \textcolor{red}{$\theta$} then \textcolor{red}{$\mathbb{P}(p(X; \theta) \leq \alpha \, | \,\theta) \leq \alpha$}. Thus, \textcolor{red}{$\mathbb{P}(\theta \in C(X; \alpha) \, | \, \theta) \geq  1 - \alpha$}. Hence, \textcolor{red}{$C(X; \alpha)$} is a level-$(1-\alpha)$ \textcolor{blue}{confidence procedure}. 
\item If \textcolor{red}{$\alpha' > \alpha$} then if \textcolor{red}{$\theta \in  C(x; \alpha')$} we have \textcolor{red}{$p(x ; \theta) > \alpha' > \alpha$} and so \textcolor{red}{$\theta \in  C(x; \alpha)$} and so \textcolor{red}{$C$} is \textcolor{blue}{nesting}.
\item If \textcolor{red}{$p$} is \textcolor{blue}{exact} then the inequalities can be replaced by equalities and so \textcolor{red}{$C$} is also \textcolor{blue}{exact}. 
\end{itemize}
We thus have 1. 
\begin{itemize}
\item Now, if \textcolor{red}{$C$} is a \textcolor{blue}{nesting} family of confidence procedures then\footnote{Here we're finessing the issue of the boundary of $C$ by assuming that if $\alpha^{*} : = \inf \{\alpha \, : \, \theta_{0} \notin C(x; \alpha)\}$ then $\theta_{0} \notin C(x; \alpha^{*})$.}
\end{itemize}
\begin{eqnarray*}
\inf \{\alpha \, : \, \theta_{0} \notin C(x; \alpha)\} \leq u & \iff & \theta_{0} \notin C(x; u).
\end{eqnarray*}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
\begin{itemize}
\item Let \textcolor{red}{$\theta_{0}$} and \textcolor{red}{$u \in [0, 1]$} be arbitrary. Then,
\begin{eqnarray*}
\mathbb{P}(p(X; \theta_{0}) \leq u \, | \, \theta_{0}) \ = \ \mathbb{P}(\theta_{0} \notin C(X; u) \, | \,\theta_{0}) \leq u
\end{eqnarray*}
as \textcolor{red}{$C(X; u)$} is a level-$(1-u)$ \textcolor{blue}{confidence procedure}. Thus, \textcolor{red}{$p$} is \textcolor{blue}{super-uniform}. 
\item If \textcolor{red}{$C$} is \textcolor{blue}{exact}, then the inequality is replaced by an equality, and hence \textcolor{red}{$p$} is \textcolor{blue}{exact} as well. \hfill $\Box$
\end{itemize}
\end{block}
\end{frame}
\subsection{Families of significance procedures}
\begin{frame}
\frametitle{Families of significance procedures}
\begin{itemize}
\item We now consider a very \textcolor{blue}{general} way to construct a family of significance procedures.
\item We will then show how to use \textcolor{blue}{simulation} to compute the family.
\end{itemize}
\begin{block}{Theorem}
Let \textcolor{red}{$t : \mathcal{X} \rightarrow \mathbb{R}$} be a statistic. For each \textcolor{red}{$x \in \mathcal{X}$}  and \textcolor{red}{$\theta_{0} \in \Theta$} define
\begin{eqnarray*}
\textcolor{red}{p_{t}(x; \theta_{0}) \ := \ \mathbb{P}(t(X) \geq t(x) \, | \, \theta_{0})}.
\end{eqnarray*}
Then \textcolor{red}{$p_{t}$} is a family of \textcolor{blue}{significance procedures}. If the distribution function of \textcolor{red}{$t(X)$} is \textcolor{blue}{continuous}, then \textcolor{red}{$p_{t}$} is \textcolor{blue}{exact}.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof (Casella and Berger, 2002)}
\begin{itemize}
\item Now 
\begin{eqnarray*}
p_{t}(x; \theta_{0}) \ = \ \mathbb{P}(t(X) \geq t(x) \, | \, \theta_{0}) \ = \ \mathbb{P}(-t(X) \leq -t(x)  \, | \, \theta_{0}).
\end{eqnarray*}
\item Let \textcolor{red}{$F$} denote the distribution function of \textcolor{red}{$Y(X) = -t(X)$} then \textcolor{red}{$p_{t}(x; \theta_{0}) = F(-t(x) \, | \, \theta_{0})$}. 
\item Assume that \textcolor{red}{$t(X)$} is \textcolor{blue}{continuous} so that $Y(X) = -t(X)$ is continuous. Using the \textcolor{blue}{Probability Integral Transform},
\begin{eqnarray*}
\textcolor{red}{\mathbb{P}(p_{t}(X; \theta_{0}) \leq \alpha \, | \, \theta_{0})} & = & \mathbb{P}(F(Y) \leq \alpha \, | \, \theta_{0}) \\ & = & \mathbb{P}(Y \leq F^{-1}(\alpha) \, | \, \theta_{0}) = F(F^{-1}(\alpha)) \textcolor{red}{= \alpha}.
\end{eqnarray*}
Hence, \textcolor{red}{$p_{t}$} is \textcolor{blue}{uniform} under \textcolor{red}{$\theta_{0}$}. 
\item It $t(X)$ is not continuous then, via the Probability Integral Transform, \textcolor{red}{$\mathbb{P}(F(Y) \leq \alpha \, | \, \theta_{0}) \leq \alpha$} and so  \textcolor{red}{$p_{t}(X; \theta_{0})$} is \textcolor{blue}{super-uniform} under \textcolor{red}{$\theta_{0}$}. \hfill $\Box$
\end{itemize} 
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item  So there is a family of significance procedures for \textcolor{blue}{each} possible function \textcolor{red}{$t : \mathcal{X} \rightarrow \mathbb{R}$}. 
\item Clearly only a tiny fraction of these can be useful functions, and the rest must be useless. 
\item Some, like \textcolor{red}{$t(x) = c$} for some constant $c$, are always useless. Others, like \textcolor{red}{$t(x) = \sin(x)$} might sometimes be a little bit useful, while others, like \textcolor{red}{$t(x) = \sum_{i} x_{i}$} might be quite useful - but it all depends on the circumstances. 
\item Some \textcolor{blue}{additional criteria} are required to separate out \textcolor{blue}{good} from \textcolor{blue}{poor} choices of the test statistic $t$, when using the construction in the theorem. 
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
The most pertinent criterion is:
\begin{itemize}
\item Select a test statistic for which \textcolor{red}{$t(X)$} which will tend to be \textcolor{blue}{larger} for decision-relevant \textcolor{blue}{departures} from \textcolor{red}{$\theta_{0}$}. 
\end{itemize}
\begin{block}{Example}
  For the likelihood ratio, $\lambda(x)$, small observed values of $\lambda(x)$ support departures from $\theta_{0}$. Thus, $t(X) = -2\log \lambda(X)$, is a test statistic for which large values support departures from $\theta_{0}$.
  \end{block}
\begin{itemize}  
\item Large values of $t(X)$ will correspond to \textcolor{blue}{small} values of the \textcolor{blue}{$p$-value}, supporting the hypothesis that $H_{1}$ is true. 
\item This criterion ensures that \textcolor{red}{$p_{t}(X; \theta_{0})$} will tend to be \textcolor{blue}{smaller} under decision-relevant departures from $\theta_{0}$; small $p$-values are more interesting, precisely because \textcolor{blue}{significance procedures} are \textcolor{blue}{super-uniform} under $\theta_{0}$.
\end{itemize}
\end{frame} 
\subsection{Computing p-values}
\begin{frame}
\frametitle{Computing p-values}
Only in very special cases will it be possible to find a \textcolor{blue}{closed-form expression} for \textcolor{red}{$p_{t}$} from which we can compute the \textcolor{blue}{$p$-value} \textcolor{red}{$p_{t}(x; \theta_{0})$}.
\begin{block}{Theorem (Adapted from Besag and Clifford, 1989)}\label{theo:besag} 
For any finite sequence of scalar random variables \textcolor{red}{$X_{0}, X_{1}, \ldots, X_{m}$}, define the \textcolor{blue}{rank} of \textcolor{red}{$X_{0}$} in the sequence as
\begin{eqnarray*}
R & := & \sum_{i=1}^{m} \mathbbm{1}_{\{X_{i} \leq X_{0}\}}.
\end{eqnarray*}
If \textcolor{red}{$X_{0}, X_{1}, \ldots, X_{m}$} are \textcolor{blue}{exchangeable}\footnote{If $X_{0}, X_{1}, \ldots, X_{m}$ are exchangeable then their joint density function satisfies
$f(x_{0}, \ldots, x_{m}) = f(x_{\pi(0)}, \ldots, x_{\pi(m)})$ for all permutations $\pi$ defined on the set $\{0, \ldots, m\}$.} then \textcolor{red}{$R$} has a \textcolor{blue}{discrete uniform distribution} on the integers \textcolor{red}{$\{0, 1, \ldots, m\}$}, and \textcolor{red}{$(R + 1)/(m + 1)$} has a \textcolor{blue}{super-uniform} distribution.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof}
By exchangeability, \textcolor{red}{$X_{0}$} has the \textcolor{blue}{same probability} of having rank \textcolor{red}{$r$} as any of the other \textcolor{red}{$X_{i}$}s, for \textcolor{blue}{any} \textcolor{red}{$r$}, and therefore
\begin{eqnarray*}
\mathbb{P}(R = r) & = & \frac{1}{m+1}
\end{eqnarray*}
for $r \in \{0, 1, \ldots, m\}$ and zero otherwise, proving the first claim. For the second claim,
$$\mathbb{P}\left(\frac{R+1}{m+1} \leq u\right) \ = \ \mathbb{P}(R+1 \leq u(m+1)) \ = \ \mathbb{P}(R +1 \leq \lfloor u(m+1) \rfloor)$$
since \textcolor{red}{$R$} is an \textcolor{blue}{integer} and \textcolor{red}{$\lfloor x \rfloor$} denotes the \textcolor{blue}{largest integer no larger} than \textcolor{red}{$x$}. 
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
Hence,
\begin{eqnarray}
\mathbb{P}\left(\frac{R+1}{m+1} \leq u\right) & = & \sum_{r=0}^{\lfloor u(m+1) \rfloor -1} \textcolor{red}{\mathbb{P}(R = r)} \label{eq:besag2} \\
& = & \sum_{r=0}^{\lfloor u(m+1) \rfloor -1} \textcolor{red}{\frac{1}{m+1}} \label{eq:besag3} \\
& = & \frac{\lfloor u(m+1) \rfloor}{m+1} \leq u, \nonumber
\end{eqnarray}
as required where equation (\ref{eq:besag3}) follows from  (\ref{eq:besag2}) by \textcolor{blue}{exchangeability}. \hfill $\Box$
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item We utilise this result to compute the \textcolor{blue}{$p$-value} \textcolor{red}{$p_{t}(x; \theta_{0})$} corresponding to the test statistic \textcolor{red}{$t(X)$} at \textcolor{red}{$\theta_{0}$}.
\item Fix the test statistic \textcolor{red}{$t(x)$} and define \textcolor{red}{$T_{i} = t(X_{i})$} where \textcolor{red}{$X_{1}, \ldots, X_{m}$ }are independent and identically distributed random variables with density \textcolor{red}{$f_{X}(\cdot \, | \, \theta_{0})$}.
\item Typically, we may have to use \textcolor{blue}{simulation} to obtain the sample and we'll need to specify \textcolor{red}{$\theta_{0}$} for this. 
\item Notice that $t(X), T_{1}, \ldots, T_{m}$ are exchangeable and thus \textcolor{red}{$-t(X), -T_{1}, \ldots, -T_{m}$} are \textcolor{blue}{exchangeable}.
\item Let 
\begin{eqnarray*}
R_{t}(x; \theta_{0}) \ := \ \sum_{i=1}^{m}  \mathbbm{1}_{\{-T_{i} \leq -t(x)\}} \ = \ \sum_{i=1}^{m}  \mathbbm{1}_{\{T_{i} \geq t(x)\}},
\end{eqnarray*}
then the previous theorem implies that
\begin{eqnarray*}
\textcolor{red}{P_{t}(x; \theta_{0}) \ := \ \frac{R_{t}(x; \theta_{0}) + 1}{m+1}}
\end{eqnarray*}
has a \textcolor{blue}{super-uniform} distribution under \textcolor{red}{$X \sim f_{X}(\cdot \, | \, \theta_{0})$}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Note that \textcolor{red}{$\mathbb{P}(T \geq t(x) \, | \,\theta_{0}) = \mathbb{E}(\mathbbm{1}_{\{T \geq t(x)\}})$}.
\item Hence, the \textcolor{blue}{Weak Law of Large Numbers (WLLN)} implies that
\begin{eqnarray*}
\lim_{m \rightarrow \infty} P_{t}(x; \theta_{0}) & = & \lim_{m \rightarrow \infty} \frac{R_{t}(x; \theta_{0}) + 1}{m+1} \\
& = & \lim_{m \rightarrow \infty} \frac{R_{t}(x; \theta_{0}) }{m} \\
& = & \textcolor{red}{\lim_{m \rightarrow \infty} \frac{\sum_{i=1}^{m}  \mathbbm{1}_{\{T_{i} \geq t(x)\}}}{m}} \\
& = & \mathbb{P}(T \geq t(x) \, | \,\theta_{0}) \ = \ p_{t}(x; \theta_{0}).
\end{eqnarray*}
\item Therefore, not only is \textcolor{red}{$P_{t}(x; \theta_{0})$} \textcolor{blue}{super-uniform} under \textcolor{red}{$\theta_{0}$}, so that $P_{t}$ is a family of significance procedures for every $m$, but the \textcolor{blue}{limiting value} of \textcolor{red}{$P_{t}(x; \theta_{0})$} as $m$ becomes large is \textcolor{red}{$p_{t}(x; \theta_{0})$}.
\item In summary, if you can \textcolor{blue}{simulate} from your model under \textcolor{red}{$\theta_{0}$} then you can produce a $p$-value for \textcolor{blue}{any test statistic} \textcolor{red}{$t$}, namely \textcolor{red}{$P_{t}(x; \theta_{0})$}, and if you can simulate cheaply, so that the number of simulations m is large, then \textcolor{red}{$P_{t}(x; \theta_{0}) \approx p_{t}(x; \theta_{0})$}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle}\ }
\begin{itemize}
\item However, this simulation-based approach is not well-adapted to constructing \textcolor{blue}{confidence sets}.
\item Let \textcolor{red}{$C_{t}$} be the family of \textcolor{blue}{confidence procedures} induced by \textcolor{red}{$p_{t}$} using \textcolor{blue}{duality}.
\item With \textcolor{blue}{one set} of $m$ simulations, we can answer "Is \textcolor{red}{$\theta_{0} \in C_{t}(x; \alpha)$}?"
\begin{itemize}
\item These simulations give a value \textcolor{red}{$P_{t}(x; \theta_{0})$} which is either larger or not larger than \textcolor{red}{$\alpha$}.
\item If \textcolor{red}{$P_{t}(x; \theta_{0}) > \alpha$} then \textcolor{red}{$\theta_{0} \in C_{t}(x; \alpha)$}, and otherwise it is not.
\end{itemize}
\item  However, this is \textcolor{blue}{not an effective way} to enumerate all of the points in \textcolor{red}{$C_{t}(x; \alpha)$} since we would need to do $m$ \textcolor{blue}{simulations} for \textcolor{blue}{each point} in \textcolor{red}{$\Theta$}.
\end{itemize}
\end{frame}
\subsection{Concluding remarks}
\begin{frame}
\frametitle{Concluding remarks}
\begin{itemize}
\item It is a very common observation, made repeatedly over the last 50 years see, for example, Rubin (1984), that clients think more like Bayesians than classicists. 
\item For example, \textcolor{red}{$\mathbb{P}(\theta \in C(X; \alpha) \, | \, \theta) \geq 1 - \alpha$} is often interpreted as a probability over \textcolor{red}{$\theta$} for the \textcolor{blue}{observed} \textcolor{red}{$C(x; \alpha)$}.
\item Classical  statisticians thus have to wrestle with the issue that their clients will likely \textcolor{blue}{misinterpret} their results.
\item This can be potentially disastrous for $p$-values. 
\begin{itemize}
\item A $p$-value \textcolor{red}{$p(x; \theta_{0})$} refers only to \textcolor{red}{$\theta_{0}$}, making \textcolor{blue}{no reference} at all to other hypotheses about \textcolor{red}{$\theta$}.
\item A posterior probability \textcolor{red}{$\pi(\theta_{0} \, | \, x)$} \textcolor{blue}{contrasts} \textcolor{red}{$\theta_{0}$} with the other values in $\Theta$ which \textcolor{red}{$\theta$} might have taken. 
\item The two outcomes can be radically different, as first captured in \textcolor{blue}{Lindley's paradox} (Lindley, 1957).
\end{itemize}
\end{itemize}
\end{frame}
\section{References}
\begin{frame}
  \frametitle{References}
  \begin{itemize}
\item  Basu, D. (1975). Statistical information and likelihood (with discussion). {\em Sankhy\={a}\/}~{\em 37\/}(1), 1--71.
\item Besag, J. and P.~Clifford (1989). Generalized {Monte Carlo} significance tests. {\em Biometrika\/} {\em 76\/}(4), 633--642.
\item Birnbaum, A. (1962). On the foundations of statistical inference. {\em Journal of the American Statistical Association\/}~{\em 57}, 269--306.
\item Birnbaum, A. (1972). More concepts of statistical evidence. {\em Journal of the American Statistical Association\/}~{\em 67}, 858--861.
\item Casella, G. and R.L.~Berger (2002). {\em Statistical Inference\/} (2nd ed.). Pacific Grove, CA, USA: Duxbury.
\item Cox, D.R. and D.V.~Hinkley (1974). {\em Theoretical Statistics}. London, UK: Chapman and Hall.
\item Dawid, A.P. (1977). Conformity of inference patterns. In J.R.~Barra et~al. (Eds.), {\em Recent Developments in Statistcs}. pp.\ 245--256. Amsterdam, The Netherlands: North-Holland Publishing Company.
\end{itemize}
\end{frame}
\begin{frame}
  \frametitle{References continued}
  \begin{itemize}
  \item  DiCiccio, T.J. and B.~Efron (1996). Bootstrap confidence intervals {\em Statistical Science\/}~{\em 11\/}(3), 189--228.
  \item Lindley, D.V. (1957). A statistical paradox. {\em Biometrika\/}~{\em 44}, 187--192.
    \item Robert, C.P. (2007). {\em The Bayesian Choice: From Decision-Theoretic Foundations to
        Computational Implementation}. New York, USA: Springer.
    \item Rubin, D.B. (1984). Bayesianly justifiable and relevant frequency calculations for the applied statistician. {\em The Annals of Statistics\/}~{\em 12\/}(4), 1151--1172.
    \item Savage, L.J. et~al. (1962). {\em The Foundations of Statistical Inference}. London, UK: Methuen.
    \item Smith, J.Q. (2010). {\em Bayesian Decision Analysis: Principle and Practice}. Cambridge, UK: Cambridge University Press.
      \item Wilks, S.S. (1938). The large-sample distribution of the likelihood ratio for testing composite hypotheses. {\em The Annals of Mathematical Statistics\/}~{\em 9}(1), 60--62.
    \end{itemize}
\end{frame}
\end{document}






