\documentclass{beamer}
\usepackage{hyperref}
% \usetheme{default}
\usetheme[secheader]{Boadilla}
%\usetheme{Hannover}
\usepackage[all]{xy}
\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage{latexsym, epsfig, subfigure , color, amsmath, bbm}
\DeclareMathOperator*{\argmin}{arg\,min}
\setbeamertemplate{blocks}[rounded][shadow]
%\setbeamercovered{transparent}
%\setbeamertemplate{navigation symbols}{}
\newcommand{\omalpha}{\ensuremath{(1 - \alpha)}\,}
\newcommand{\link}[1]{\underline{\color{red}#1}}
\newcommand{\hilite}[1]{{\color{red}#1}}
\newcommand{\Ev}{\ensuremath{\mbox{Ev}}}
\newcommand{\boost}{\vphantom{\big(}}
\newenvironment{variableblock}[3]{%
  \setbeamercolor{block body}{#2}
  \setbeamercolor{block title}{#3}
  \begin{block}{#1}}{\end{block}}
\definecolor{light-blue}{rgb}{0.29, 0.59, 0.82}
\title{Statistical Inference \\ Lecture Six}
\subtitle{\href{https://people.bath.ac.uk/masss/APTS/2022-23/LectureSix.pdf}{https://people.bath.ac.uk/masss/APTS/2022-23/LectureSix.pdf}}
\author{Simon Shaw}
\institute{University of Bath}
\date{APTS, 13-16 December 2022}

\begin{document}
\begin{frame}
\titlepage
\end{frame}
\section{Confidence sets and p-values}
\subsection{Overview of Lecture Six}
\begin{frame}
  \frametitle{Overview of Lecture Six}
 In Lecture Five we introduced confidence procedures.
  \begin{itemize}
\item Confidence procedure: A \textcolor{blue}{random set} \textcolor{red}{$C(X) \subset \Theta$} is a level-\omalpha \textcolor{blue}{confidence procedure} exactly when $\textcolor{red}{\mathbb{P}(\theta \in C(X) \, | \, \theta) \ \geq \ 1 - \alpha}$.
\item \textcolor{blue}{Family of confidence procedures}: occurs when \textcolor{red}{$C(X; \alpha)$} is a level-$(1-\alpha)$ confidence procedure for \textcolor{blue}{every} \textcolor{red}{$\alpha \in [0, 1]$}. 
\item \textcolor{blue}{Level set property, LSP}: present for a \textcolor{blue}{confidence procedure} $C$ when \textcolor{red}{$C(x) = \{\theta \, : \, f_{X}(x \, | \, \theta) > g(x)\}$} for some \textcolor{red}{$g : \mathcal{X} \rightarrow \mathbb{R}$}.
\end{itemize}
In Lecture Six we'll look at good choices of confidence procedures.
\begin{itemize}
\item For the \textcolor{blue}{linear model} we can construct an exact family of confidence procedures which satisfy the LSP.
\item \textcolor{blue}{Wilks Confidence procedures} and the likelihood ratio test.
\item Introduce the $p$-value.
\end{itemize}
 \end{frame}
\begin{frame}
\frametitle{Overview of Lecture Six continued}
  \begin{itemize}
\item A $p$-value \textcolor{red}{$p(X)$} is a \textcolor{blue}{statistic} satisfying, for every $\alpha \in [0, 1]$, \textcolor{red}{$\mathbb{P}(p(X) \leq \alpha \, | \, \theta) \leq \alpha$}. It is \textcolor{blue}{super-uniform}.
\item \textcolor{red}{$p : \mathcal{X} \rightarrow \mathbb{R}$} is a \textcolor{blue}{significance procedure} for \textcolor{red}{$\theta_{0} \in \Theta$} exactly when \textcolor{red}{$p(X)$} is \textcolor{blue}{super-uniform} under \textcolor{red}{$\theta_{0}$}. 
\item We'll show there is a \textcolor{blue}{duality} between \textcolor{blue}{significance procedures} and \textcolor{blue}{confidence procedures}.
\item We'll show how to construct a family of significance procedures  and how to use simulation to compute the family.
\end{itemize}
 \end{frame} 
 \subsection{Good choices of confidence procedures}

  
  \begin{frame}
%\frametitle{\ }
$\textcolor{red}{C_{h}(x; \alpha) \ := \  \{ \theta \in \Theta \, : \, f_{X}(x \, | \, \theta) > \alpha h(x)\}}$, where \textcolor{red}{$h$} is any \textcolor{blue}{probability density function} for \textcolor{red}{$X$}, is a family of confidence procedures, with the LSP.
\begin{itemize}
\item Among the interesting choices for \textcolor{red}{$h$}, one possibility is \textcolor{red}{$h(x) =f_{X}(x \, | \, \theta_{0})$}, for some \textcolor{red}{$\theta_{0} \in \Theta$}. 
\item As \textcolor{red}{$f_{X}(x \, | \, \theta_{0}) > \alpha f_{X}(x \, | \, \theta_{0})$} we can \textcolor{blue}{construct} a level-\textcolor{red}{\omalpha} \textcolor{blue}{confidence procedure} whose confidence sets will \textcolor{blue}{always} contain \textcolor{red}{$\theta_{0}$}. 
\item This suggests an issue with confidence procedures: two statisticians may come to two \textcolor{blue}{different} conclusions about \textcolor{red}{$H_0 : \theta = \theta_0$} depending on the intervals \textcolor{blue}{they construct}. 
\item This illustrates why it is important to be able to \textcolor{blue}{account} for the \textcolor{blue}{choices} you make as a statistician.
\item The theorem utilises Markov's Inequality which is a \textcolor{blue}{very slack} result. It is likely that the \textcolor{blue}{coverage} of the corresponding family of confidence procedures will be \textcolor{blue}{much larger} than \textcolor{red}{\omalpha}.
\item A more desirable strategy would be to use an \textcolor{blue}{exact family} of confidence procedures which satisfy the \textcolor{blue}{LSP}, if one existed.
\end{itemize}
\end{frame}
\subsection{The linear model}
\begin{frame}
\frametitle{The linear model}
\begin{itemize}
\item We'll briefly discuss the \textcolor{blue}{linear model} and construct an \textcolor{blue}{exact family} of confidence procedures which satisfy the \textcolor{blue}{LSP}.
\item Let \textcolor{red}{$Y = (Y_{1}, \ldots, Y_{n})$} be an \textcolor{red}{$n$}-vector of observables with \textcolor{red}{$Y = X\theta + \epsilon$}.
\begin{itemize}
\item  \textcolor{red}{$X$} is an \textcolor{red}{$(n \times p)$} matrix\footnote{We typically use $X$ to denote a generic random variable and so it is not ideal to use it here for a specified matrix but this is the standard notation for linear models.} of \textcolor{blue}{regressors}, 
\item \textcolor{red}{$\theta$} is a \textcolor{red}{$p$}-vector of \textcolor{blue}{regression coefficients},
\item \textcolor{red}{$\epsilon$} is an \textcolor{red}{$n$}-vector of \textcolor{blue}{residuals}. 
\end{itemize}
\item Assume that \textcolor{red}{$\epsilon \sim N_{n}(0, \sigma^{2}I_{n})$}, the $n$-dimensional \textcolor{blue}{multivariate normal} distribution, where \textcolor{red}{$\sigma^{2}$} is \textcolor{blue}{known} and \textcolor{red}{$I_{n}$} is the $(n \times n)$ \textcolor{blue}{identity matrix}.
\item From properties of the multivariate normal distribution, it follows that \textcolor{red}{$Y \sim N_{n}(X\theta, \sigma^{2}I_{n})$}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
Now,
\begin{eqnarray*}
  L_{Y}(\theta; y) & = & \left(2\pi \sigma^{2}\right)^{-\frac{n}{2}} \exp\left\{-\frac{1}{2\sigma^{2}} (y - X\theta)^{T}(y-X\theta)\right\}. \label{eq:linear}
\end{eqnarray*}
Let \textcolor{red}{$\hat{\theta} = \hat{\theta}(y) = \left(X^{T}X\right)^{-1}X^{T}y$} then
\begin{eqnarray*}
  \textcolor{red}{(y - X\theta)^{T}(y-X\theta)} & = & (y-X\hat{\theta} + X\hat{\theta} - X\theta)^{T}(y-X\hat{\theta} + X\hat{\theta} - X\theta) \nonumber \\
                               & = & (y-X\hat{\theta})^{T}(y-X\hat{\theta}) + (X\hat{\theta} - X\theta)^{T}(X\hat{\theta} - X\theta) \nonumber \\
                               & = & \textcolor{red}{(y-X\hat{\theta})^{T}(y-X\hat{\theta}) + (\hat{\theta} - \theta)^{T}X^{T}X(\hat{\theta} - \theta)}. \label{eq:linearI}
\end{eqnarray*}
Thus, \textcolor{red}{$(y - X\theta)^{T}(y-X\theta)$} is \textcolor{blue}{minimised} when \textcolor{red}{$\theta = \hat{\theta}$} and so, \textcolor{red}{$\hat{\theta} = \left(X^{T}X\right)^{-1}X^{T}y$} is the \textcolor{blue}{mle} of \textcolor{red}{$\theta$}. The likelihood ratio is 
\begin{eqnarray*}
  \lambda(y) & = & \frac{L_{Y}(\theta; y)}{L_{Y}(\hat{\theta}; y)}\nonumber \\
             & = & \exp\left\{-\frac{1}{2\sigma^{2}}\left[ \textcolor{red}{(y - X\theta)^{T}(y-X\theta) - (y - X\hat{\theta})^{T}(y-X\hat{\theta})}\right]\right\} \nonumber\\
             & = & \exp\left\{-\frac{1}{2\sigma^{2}}\textcolor{red}{(\hat{\theta} - \theta)^{T}X^{T}X(\hat{\theta} - \theta)}\right\}\label{eq:linearII}
\end{eqnarray*}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Thus,
  \textcolor{red}{$-2\log \lambda(y)  = \frac{1}{\sigma^{2}}(\hat{\theta} - \theta)^{T}X^{T}X(\hat{\theta} - \theta)$}.
\item As \textcolor{red}{$\hat{\theta}(Y) = \left(X^{T}X\right)^{-1}X^{T}Y$} then, as \textcolor{red}{$Y \sim N_{n}(X\theta, \sigma^{2}I_{n})$}, $$\hat{\theta}(Y) \sim N_{p}\left(\theta, \sigma^{2}\left(X^{T}X\right)^{-1}\right)$$
\item Consequently, \textcolor{red}{$-2\log \lambda(Y) \sim \chi_{p}^{2}$}. \end{itemize}
Hence, with $\mathbb{P}(\chi^{2}_{p} \geq \chi^{2}_{p, \alpha}) = \alpha$,
\begin{eqnarray*}
  \textcolor{red}{C(y; \alpha)} & = & \left\{\theta \in \mathbb{R}^{p} \, : \, -2\log\lambda(y) \ = \ -2\log\frac{f_{Y}(y \, | \, \theta, \sigma^{2})}{f_{Y}(y \, | \, \hat{\theta}, \sigma^{2})} < \chi^{2}_{p, \alpha}\right\} \nonumber\\
               & = & \textcolor{red}{\left\{\theta \in \mathbb{R}^{p} \, : \, f_{Y}(y \, | \, \theta, \sigma^{2}) > \exp\left(-\frac{\chi^{2}_{p, \alpha}}{2}\right)f_{Y}(y \, | \, \hat{\theta}, \sigma^{2})\right\}} \label{eq:wilks}
\end{eqnarray*}
is a family of \textcolor{blue}{exact confidence procedures} for \textcolor{red}{$\theta$} which has the \textcolor{blue}{LSP}.
\end{frame}
\subsection{Wilks confidence procedures}
\begin{frame}
\frametitle{Wilks confidence procedures}
\begin{itemize}
\item This outcome, where we can find a family of exact confidence procedures with the LSP, is \textcolor{blue}{more-or-less unique} to the regression parameters of the \textcolor{blue}{linear model}.
\item  It is however found, \textcolor{blue}{approximately}, in the \textcolor{blue}{large} \textcolor{red}{$n$} behaviour of a much wider class of models. 
\end{itemize}
\begin{block}{Wilks' Theorem}
  Let $X = (X_{1}, \ldots, X_{n})$  where each $X_{i}$ is independent and identically distributed, \textcolor{red}{$X_{i} \sim f(x_{i} \, | \, \theta)$}, where $f$ is a \textcolor{blue}{regular model} and the \textcolor{blue}{parameter space} $\Theta$ is an open convex subset of \textcolor{red}{$\mathbb{R}^{p}$} (and invariant to $n$). The distribution of the statistic \textcolor{red}{$-2\log \lambda(X)$} converges to a \textcolor{blue}{chi-squared} distribution with \textcolor{red}{$p$} degrees of freedom as $n \rightarrow \infty$.
  \end{block}
  \begin{itemize}
  \item A working guideline to regular model is that $f$ must be smooth and differentiable in $\theta$; in particular, the support must not depend on $\theta$.  
  \end{itemize}
   \end{frame} 
  \begin{frame}
  %\frametitle{\ }
  \begin{itemize}
   \item The result dates back to Wilks (1938) and, as such, the resultant confidence procedures are often termed \textcolor{blue}{Wilks confidence procedures}.
  \item  Thus, if the conditions of Wilks' Theorem are met, 
   \begin{eqnarray*}
    \textcolor{red}{C(x; \alpha) \ = \ \left\{\theta \in \mathbb{R}^{p} \, : \, f_{X}(x \, | \, \theta) > \exp\left(-\frac{\chi^{2}_{p, \alpha}}{2}\right)f_{X}(x \, | \, \hat{\theta})\right\}} \label{eq:wilksI}
  \end{eqnarray*}
  is a family of \textcolor{blue}{approximately exact} confidence procedures which satisfy the LSP. 
  \item For a given model, the pertinent question is whether or not the approximation is a good one.
  \item We are thus interested in the \textcolor{blue}{level error}, the difference between the \textcolor{blue}{nominal level}, typically $(1 - \alpha)$ everywhere, and the \textcolor{blue}{actual level}, the actual minimum coverage everywhere,
  \begin{eqnarray*}
  \mbox{level error} & = &  \mbox{nominal level} - \mbox{actual level}.
\end{eqnarray*}
  \item Methods, such as \textcolor{blue}{bootstrap calibration}, described in DiCiccio and Efron (1996), exist which attempt to \textcolor{blue}{correct} for the level error.
  \end{itemize}
\end{frame}
\subsection{Significance procedures and duality}
\begin{frame}
\frametitle{Significance procedures and duality}
\begin{itemize}
\item  A \textcolor{blue}{hypothesis test} of \textcolor{red}{$H_{0}: \theta \in \Theta_{0}$} versus \textcolor{red}{$H_{1}: \theta \in \Theta_{0}^{\rm c}$}, where $\Theta_{0} \cup \Theta_{0}^{\rm c} = \Theta$, at significance level of $5\%$ (or any other specified value) returns \textcolor{blue}{one} bit of information, either we \textcolor{blue}{accept $H_{0}$} or \textcolor{blue}{reject $H_{0}$}. 
\item  We do not know whether the decision was \textcolor{blue}{borderline} or nearly \textcolor{blue}{conclusive}; i.e.\ whether, for rejection, $H_0$ and $C(x; 0.05)$ were close, or well-separated. 
\item Of more interest is to consider the \textcolor{blue}{smallest} value of \textcolor{red}{$\alpha$} for which \textcolor{red}{$C(x; \alpha)$} does \textcolor{blue}{not} intersect \textcolor{red}{$H_{0}$}. This value is termed the \textcolor{blue}{$p$-value}. 
\end{itemize}
\begin{block}{Definition ($p$-value)} 
A $p$-value \textcolor{red}{$p(X)$} is a \textcolor{blue}{statistic} satisfying \textcolor{red}{$p(x) \in [0,1]$} for every $x \in \mathcal{X}$. \textcolor{blue}{Small} values of \textcolor{red}{$p(x)$} support the hypothesis that \textcolor{red}{$H_{1}$} is true. A $p$-value is valid if, for every $\theta \in \Theta_{0}$ and every $\alpha \in [0, 1]$,
\begin{eqnarray*}
\mathbb{P}(p(X) \leq \alpha \, | \, \theta) & \leq & \alpha. \label{eq:pvalue}
\end{eqnarray*}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item If $p(X)$ is a valid $p$-value then a \textcolor{blue}{significance test} that rejects $H_{0}$ if and only if $p(X) \leq \alpha$ is a test with \textcolor{blue}{significance level} $\alpha$. 
\item In this part we introduce the idea of \textcolor{blue}{significance procedure} at level \textcolor{red}{$\alpha$}, deriving a \textcolor{blue}{duality} between it and a level \textcolor{red}{$1-\alpha$} \textcolor{blue}{confidence procedure}.  
\item Let $X$ and $Y$ be two \textcolor{blue}{scalar} random variables.  Then \textcolor{red}{$X$} \textcolor{blue}{stochastically dominates} \textcolor{red}{$Y$} exactly when \textcolor{red}{$\mathbb{P}(X \leq v) \leq \mathbb{P}(Y \leq v)$} for all \textcolor{red}{$v \in \mathbb{R}$}.
\item If \textcolor{red}{$U \sim \mbox{Unif}(0, 1)$} then \textcolor{red}{$\mathbb{P}(U \leq u)  = u$} for \textcolor{red}{$u \in [0, 1]$}. With this in mind, we make the following definition.
\end{itemize}
\begin{block}{Definition (Super-uniform)}
The random variable $X$ is \textcolor{blue}{super-uniform} exactly when it \textcolor{blue}{stochastically dominates} a standard \textcolor{blue}{uniform} random variable. That is
\begin{eqnarray*}
\mathbb{P}(X \leq u) & \leq & u \label{fn:sunif}
\end{eqnarray*}
for all $u \in [0, 1]$.
\end{block}
\begin{itemize}
\item Thus, for \textcolor{red}{$\theta \in \Theta_{0}$}, the \textcolor{blue}{$p$-value} \textcolor{red}{$p(X)$} is \textcolor{blue}{super-uniform}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item We now define a significance procedure. Note the similarities with the definitions of a confidence procedure which are not coincidental.
\end{itemize}
\begin{block}{Definition (Significance procedure)}
\begin{enumerate}
\item \textcolor{red}{$p : \mathcal{X} \rightarrow \mathbb{R}$} is a \textcolor{blue}{significance procedure} for \textcolor{red}{$\theta_{0} \in \Theta$} exactly when \textcolor{red}{$p(X)$} is \textcolor{blue}{super-uniform} under \textcolor{red}{$\theta_{0}$}. If \textcolor{red}{$p(X)$} is \textcolor{blue}{uniform} under $\theta_{0}$, then p is an \textcolor{blue}{exact} significance procedure for $\theta_{0}$. 
\item For \textcolor{red}{$X = x$}, \textcolor{red}{$p(x)$} is a \textcolor{blue}{significance level} or (observed) $p$-value for \textcolor{red}{$\theta_{0}$} exactly when \textcolor{red}{$p$} is a \textcolor{blue}{significance procedure} for \textcolor{red}{$\theta_{0}$}. 
\item \textcolor{red}{$p : \mathcal{X} \times \Theta \rightarrow \mathbb{R}$} is a \textcolor{blue}{family of significance procedures} exactly when \textcolor{red}{$p(x; \theta_{0})$} is a \textcolor{blue}{significance procedure} for \textcolor{red}{$\theta_{0}$} for \textcolor{blue}{every} \textcolor{red}{$\theta_{0} \in \Theta$}.
\end{enumerate}
\end{block}
\begin{itemize}
\item We now show that there is a duality between significance procedures and confidence procedures.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Duality Theorem}
\begin{enumerate}
\item Let \textcolor{red}{$p$} be a family of \textcolor{blue}{significance procedures}. Then
\begin{eqnarray*}
\textcolor{red}{C(x; \alpha) \ := \ \{\theta \in \Theta \, : \, p(x; \theta) > \alpha\}}
\end{eqnarray*}
is a nesting family of \textcolor{blue}{confidence procedures}.
\item Conversely, let \textcolor{red}{$C$} be a nesting family of \textcolor{blue}{confidence procedures}. Then
\begin{eqnarray*}
\textcolor{red}{p(x; \theta_{0}) \ := \ \inf \{\alpha \, : \, \theta_{0} \notin C(x; \alpha)\}}
\end{eqnarray*}
is a family of \textcolor{blue}{significance procedures}.
\end{enumerate}
If \textcolor{blue}{either} is \textcolor{blue}{exact}, then the \textcolor{blue}{other} is \textcolor{blue}{exact} as well.
\end{block} 
\begin{block}{Proof} \begin{itemize} \item If $p$ is a family of significance procedures then for any $\theta \in \Theta$,
$$\mathbb{P}(\theta \in C(X; \alpha) \, | \, \theta) \ = \ \mathbb{P}(p(X; \theta) > \alpha \, | \,\theta) \ = \ 1 - \mathbb{P}(p(X; \theta) \leq \alpha \, | \,\theta). $$ 
\end{itemize}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
\begin{itemize}
\item Now, as \textcolor{red}{$p$} is \textcolor{blue}{super-uniform} for \textcolor{red}{$\theta$} then \textcolor{red}{$\mathbb{P}(p(X; \theta) \leq \alpha \, | \,\theta) \leq \alpha$}. Thus, \textcolor{red}{$\mathbb{P}(\theta \in C(X; \alpha) \, | \, \theta) \geq  1 - \alpha$}. Hence, \textcolor{red}{$C(X; \alpha)$} is a level-$(1-\alpha)$ \textcolor{blue}{confidence procedure}. 
\item If \textcolor{red}{$\alpha' > \alpha$} then if \textcolor{red}{$\theta \in  C(x; \alpha')$} we have \textcolor{red}{$p(x ; \theta) > \alpha' > \alpha$} and so \textcolor{red}{$\theta \in  C(x; \alpha)$} and so \textcolor{red}{$C$} is \textcolor{blue}{nesting}.
\item If \textcolor{red}{$p$} is \textcolor{blue}{exact} then the inequalities can be replaced by equalities and so \textcolor{red}{$C$} is also \textcolor{blue}{exact}. 
\end{itemize}
We thus have 1. 
\begin{itemize}
\item Now, if \textcolor{red}{$C$} is a \textcolor{blue}{nesting} family of confidence procedures then\footnote{Here we're finessing the issue of the boundary of $C$ by assuming that if $\alpha^{*} : = \inf \{\alpha \, : \, \theta_{0} \notin C(x; \alpha)\}$ then $\theta_{0} \notin C(x; \alpha^{*})$.}
\end{itemize}
\begin{eqnarray*}
\inf \{\alpha \, : \, \theta_{0} \notin C(x; \alpha)\} \leq u & \iff & \theta_{0} \notin C(x; u).
\end{eqnarray*}
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
\begin{itemize}
\item Let \textcolor{red}{$\theta_{0}$} and \textcolor{red}{$u \in [0, 1]$} be arbitrary. Then,
\begin{eqnarray*}
\mathbb{P}(p(X; \theta_{0}) \leq u \, | \, \theta_{0}) \ = \ \mathbb{P}(\theta_{0} \notin C(X; u) \, | \,\theta_{0}) \leq u
\end{eqnarray*}
as \textcolor{red}{$C(X; u)$} is a level-$(1-u)$ \textcolor{blue}{confidence procedure}. Thus, \textcolor{red}{$p$} is \textcolor{blue}{super-uniform}. 
\item If \textcolor{red}{$C$} is \textcolor{blue}{exact}, then the inequality is replaced by an equality, and hence \textcolor{red}{$p$} is \textcolor{blue}{exact} as well. \hfill $\Box$
\end{itemize}
\end{block}
\end{frame}
\subsection{Families of significance procedures}
\begin{frame}
\frametitle{Families of significance procedures}
\begin{itemize}
\item We now consider a very \textcolor{blue}{general} way to construct a family of significance procedures.
\item We will then show how to use \textcolor{blue}{simulation} to compute the family.
\end{itemize}
\begin{block}{Theorem}
Let \textcolor{red}{$t : \mathcal{X} \rightarrow \mathbb{R}$} be a statistic. For each \textcolor{red}{$x \in \mathcal{X}$}  and \textcolor{red}{$\theta_{0} \in \Theta$} define
\begin{eqnarray*}
\textcolor{red}{p_{t}(x; \theta_{0}) \ := \ \mathbb{P}(t(X) \geq t(x) \, | \, \theta_{0})}.
\end{eqnarray*}
Then \textcolor{red}{$p_{t}$} is a family of \textcolor{blue}{significance procedures}. If the distribution function of \textcolor{red}{$t(X)$} is \textcolor{blue}{continuous}, then \textcolor{red}{$p_{t}$} is \textcolor{blue}{exact}.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof (Casella and Berger, 2002)}
\begin{itemize}
\item Now 
\begin{eqnarray*}
p_{t}(x; \theta_{0}) \ = \ \mathbb{P}(t(X) \geq t(x) \, | \, \theta_{0}) \ = \ \mathbb{P}(-t(X) \leq -t(x)  \, | \, \theta_{0}).
\end{eqnarray*}
\item Let \textcolor{red}{$F$} denote the distribution function of \textcolor{red}{$Y(X) = -t(X)$} then \textcolor{red}{$p_{t}(x; \theta_{0}) = F(-t(x) \, | \, \theta_{0})$}. 
\item Assume that \textcolor{red}{$t(X)$} is \textcolor{blue}{continuous} so that $Y(X) = -t(X)$ is continuous. Using the \textcolor{blue}{Probability Integral Transform},
\begin{eqnarray*}
\textcolor{red}{\mathbb{P}(p_{t}(X; \theta_{0}) \leq \alpha \, | \, \theta_{0})} & = & \mathbb{P}(F(Y) \leq \alpha \, | \, \theta_{0}) \\ & = & \mathbb{P}(Y \leq F^{-1}(\alpha) \, | \, \theta_{0}) = F(F^{-1}(\alpha)) \textcolor{red}{= \alpha}.
\end{eqnarray*}
Hence, \textcolor{red}{$p_{t}$} is \textcolor{blue}{uniform} under \textcolor{red}{$\theta_{0}$}. 
\item It $t(X)$ is not continuous then, via the Probability Integral Transform, \textcolor{red}{$\mathbb{P}(F(Y) \leq \alpha \, | \, \theta_{0}) \leq \alpha$} and so  \textcolor{red}{$p_{t}(X; \theta_{0})$} is \textcolor{blue}{super-uniform} under \textcolor{red}{$\theta_{0}$}. \hfill $\Box$
\end{itemize} 
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item  So there is a family of significance procedures for \textcolor{blue}{each} possible function \textcolor{red}{$t : \mathcal{X} \rightarrow \mathbb{R}$}. 
\item Clearly only a tiny fraction of these can be useful functions, and the rest must be useless. 
\item Some, like \textcolor{red}{$t(x) = c$} for some constant $c$, are always useless. Others, like \textcolor{red}{$t(x) = \sin(x)$} might sometimes be a little bit useful, while others, like \textcolor{red}{$t(x) = \sum_{i} x_{i}$} might be quite useful - but it all depends on the circumstances. 
\item Some \textcolor{blue}{additional criteria} are required to separate out \textcolor{blue}{good} from \textcolor{blue}{poor} choices of the test statistic $t$, when using the construction in the theorem. 
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
The most pertinent criterion is:
\begin{itemize}
\item Select a test statistic for which \textcolor{red}{$t(X)$} which will tend to be \textcolor{blue}{larger} for decision-relevant \textcolor{blue}{departures} from \textcolor{red}{$\theta_{0}$}. 
\end{itemize}
\begin{block}{Example}
  For the likelihood ratio, $\lambda(x)$, small observed values of $\lambda(x)$ support departures from $\theta_{0}$. Thus, $t(X) = -2\log \lambda(X)$, is a test statistic for which large values support departures from $\theta_{0}$.
  \end{block}
\begin{itemize}  
\item Large values of $t(X)$ will correspond to \textcolor{blue}{small} values of the \textcolor{blue}{$p$-value}, supporting the hypothesis that $H_{1}$ is true. 
\item This criterion ensures that \textcolor{red}{$p_{t}(X; \theta_{0})$} will tend to be \textcolor{blue}{smaller} under decision-relevant departures from $\theta_{0}$; small $p$-values are more interesting, precisely because \textcolor{blue}{significance procedures} are \textcolor{blue}{super-uniform} under $\theta_{0}$.
\end{itemize}
\end{frame} 
\subsection{Computing p-values}
\begin{frame}
\frametitle{Computing p-values}
Only in very special cases will it be possible to find a \textcolor{blue}{closed-form expression} for \textcolor{red}{$p_{t}$} from which we can compute the \textcolor{blue}{$p$-value} \textcolor{red}{$p_{t}(x; \theta_{0})$}.
\begin{block}{Theorem (Adapted from Besag and Clifford, 1989)}\label{theo:besag} 
For any finite sequence of scalar random variables \textcolor{red}{$X_{0}, X_{1}, \ldots, X_{m}$}, define the \textcolor{blue}{rank} of \textcolor{red}{$X_{0}$} in the sequence as
\begin{eqnarray*}
R & := & \sum_{i=1}^{m} \mathbbm{1}_{\{X_{i} \leq X_{0}\}}.
\end{eqnarray*}
If \textcolor{red}{$X_{0}, X_{1}, \ldots, X_{m}$} are \textcolor{blue}{exchangeable}\footnote{If $X_{0}, X_{1}, \ldots, X_{m}$ are exchangeable then their joint density function satisfies
$f(x_{0}, \ldots, x_{m}) = f(x_{\pi(0)}, \ldots, x_{\pi(m)})$ for all permutations $\pi$ defined on the set $\{0, \ldots, m\}$.} then \textcolor{red}{$R$} has a \textcolor{blue}{discrete uniform distribution} on the integers \textcolor{red}{$\{0, 1, \ldots, m\}$}, and \textcolor{red}{$(R + 1)/(m + 1)$} has a \textcolor{blue}{super-uniform} distribution.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof}
By exchangeability, \textcolor{red}{$X_{0}$} has the \textcolor{blue}{same probability} of having rank \textcolor{red}{$r$} as any of the other \textcolor{red}{$X_{i}$}s, for \textcolor{blue}{any} \textcolor{red}{$r$}, and therefore
\begin{eqnarray*}
\mathbb{P}(R = r) & = & \frac{1}{m+1}
\end{eqnarray*}
for $r \in \{0, 1, \ldots, m\}$ and zero otherwise, proving the first claim. For the second claim,
$$\mathbb{P}\left(\frac{R+1}{m+1} \leq u\right) \ = \ \mathbb{P}(R+1 \leq u(m+1)) \ = \ \mathbb{P}(R +1 \leq \lfloor u(m+1) \rfloor)$$
since \textcolor{red}{$R$} is an \textcolor{blue}{integer} and \textcolor{red}{$\lfloor x \rfloor$} denotes the \textcolor{blue}{largest integer no larger} than \textcolor{red}{$x$}. 
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
Hence,
\begin{eqnarray}
\mathbb{P}\left(\frac{R+1}{m+1} \leq u\right) & = & \sum_{r=0}^{\lfloor u(m+1) \rfloor -1} \textcolor{red}{\mathbb{P}(R = r)} \label{eq:besag2} \\
& = & \sum_{r=0}^{\lfloor u(m+1) \rfloor -1} \textcolor{red}{\frac{1}{m+1}} \label{eq:besag3} \\
& = & \frac{\lfloor u(m+1) \rfloor}{m+1} \leq u, \nonumber
\end{eqnarray}
as required where equation (\ref{eq:besag3}) follows from  (\ref{eq:besag2}) by \textcolor{blue}{exchangeability}. \hfill $\Box$
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item We utilise this result to compute the \textcolor{blue}{$p$-value} \textcolor{red}{$p_{t}(x; \theta_{0})$} corresponding to the test statistic \textcolor{red}{$t(X)$} at \textcolor{red}{$\theta_{0}$}.
\item Fix the test statistic \textcolor{red}{$t(x)$} and define \textcolor{red}{$T_{i} = t(X_{i})$} where \textcolor{red}{$X_{1}, \ldots, X_{m}$ }are independent and identically distributed random variables with density \textcolor{red}{$f_{X}(\cdot \, | \, \theta_{0})$}.
\item Typically, we may have to use \textcolor{blue}{simulation} to obtain the sample and we'll need to specify \textcolor{red}{$\theta_{0}$} for this. 
\item Notice that $t(X), T_{1}, \ldots, T_{m}$ are exchangeable and thus \textcolor{red}{$-t(X), -T_{1}, \ldots, -T_{m}$} are \textcolor{blue}{exchangeable}.
\item Let 
\begin{eqnarray*}
R_{t}(x; \theta_{0}) \ := \ \sum_{i=1}^{m}  \mathbbm{1}_{\{-T_{i} \leq -t(x)\}} \ = \ \sum_{i=1}^{m}  \mathbbm{1}_{\{T_{i} \geq t(x)\}},
\end{eqnarray*}
then the previous theorem implies that
\begin{eqnarray*}
\textcolor{red}{P_{t}(x; \theta_{0}) \ := \ \frac{R_{t}(x; \theta_{0}) + 1}{m+1}}
\end{eqnarray*}
has a \textcolor{blue}{super-uniform} distribution under \textcolor{red}{$X \sim f_{X}(\cdot \, | \, \theta_{0})$}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Note that \textcolor{red}{$\mathbb{P}(T \geq t(x) \, | \,\theta_{0}) = \mathbb{E}(\mathbbm{1}_{\{T \geq t(x)\}})$}.
\item Hence, the \textcolor{blue}{Weak Law of Large Numbers (WLLN)} implies that
\begin{eqnarray*}
\lim_{m \rightarrow \infty} P_{t}(x; \theta_{0}) & = & \lim_{m \rightarrow \infty} \frac{R_{t}(x; \theta_{0}) + 1}{m+1} \\
& = & \lim_{m \rightarrow \infty} \frac{R_{t}(x; \theta_{0}) }{m} \\
& = & \textcolor{red}{\lim_{m \rightarrow \infty} \frac{\sum_{i=1}^{m}  \mathbbm{1}_{\{T_{i} \geq t(x)\}}}{m}} \\
& = & \mathbb{P}(T \geq t(x) \, | \,\theta_{0}) \ = \ p_{t}(x; \theta_{0}).
\end{eqnarray*}
\item Therefore, not only is \textcolor{red}{$P_{t}(x; \theta_{0})$} \textcolor{blue}{super-uniform} under \textcolor{red}{$\theta_{0}$}, so that $P_{t}$ is a family of significance procedures for every $m$, but the \textcolor{blue}{limiting value} of \textcolor{red}{$P_{t}(x; \theta_{0})$} as $m$ becomes large is \textcolor{red}{$p_{t}(x; \theta_{0})$}.
\item In summary, if you can \textcolor{blue}{simulate} from your model under \textcolor{red}{$\theta_{0}$} then you can produce a $p$-value for \textcolor{blue}{any test statistic} \textcolor{red}{$t$}, namely \textcolor{red}{$P_{t}(x; \theta_{0})$}, and if you can simulate cheaply, so that the number of simulations m is large, then \textcolor{red}{$P_{t}(x; \theta_{0}) \approx p_{t}(x; \theta_{0})$}.
\end{itemize}
\end{frame}
\end{document}
\begin{frame}
%\frametitle}\ }
\begin{itemize}
\item However, this simulation-based approach is not well-adapted to constructing \textcolor{blue}{confidence sets}.
\item Let \textcolor{red}{$C_{t}$} be the family of \textcolor{blue}{confidence procedures} induced by \textcolor{red}{$p_{t}$} using \textcolor{blue}{duality}.
\item With \textcolor{blue}{one set} of $m$ simulations, we can answer "Is \textcolor{red}{$\theta_{0} \in C_{t}(x; \alpha)$}?"
\begin{itemize}
\item These simulations give a value \textcolor{red}{$P_{t}(x; \theta_{0})$} which is either larger or not larger than \textcolor{red}{$\alpha$}.
\item If \textcolor{red}{$P_{t}(x; \theta_{0}) > \alpha$} then \textcolor{red}{$\theta_{0} \in C_{t}(x; \alpha)$}, and otherwise it is not.
\end{itemize}
\item  However, this is \textcolor{blue}{not an effective way} to enumerate all of the points in \textcolor{red}{$C_{t}(x; \alpha)$} since we would need to do $m$ \textcolor{blue}{simulations} for \textcolor{blue}{each point} in \textcolor{red}{$\Theta$}.
\end{itemize}
\end{frame}
\subsection{Concluding remarks}
\begin{frame}
\frametitle{Concluding remarks}
\begin{itemize}
\item It is a very common observation, made repeatedly over the last 50 years see, for example, Rubin (1984), that clients think more like Bayesians than classicists. 
\item For example, \textcolor{red}{$\mathbb{P}(\theta \in C(X; \alpha) \, | \, \theta) \geq 1 - \alpha$} is often interpreted as a probability over \textcolor{red}{$\theta$} for the \textcolor{blue}{observed} \textcolor{red}{$C(x; \alpha)$}.
\item Classical  statisticians thus have to wrestle with the issue that their clients will likely \textcolor{blue}{misinterpret} their results.
\item This can be potentially disastrous for $p$-values. 
\begin{itemize}
\item A $p$-value \textcolor{red}{$p(x; \theta_{0})$} refers only to \textcolor{red}{$\theta_{0}$}, making \textcolor{blue}{no reference} at all to other hypotheses about \textcolor{red}{$\theta$}.
\item A posterior probability \textcolor{red}{$\pi(\theta_{0} \, | \, x)$} \textcolor{blue}{contrasts} \textcolor{red}{$\theta_{0}$} with the other values in $\Theta$ which \textcolor{red}{$\theta$} might have taken. 
\item The two outcomes can be radically different, as first captured in \textcolor{blue}{Lindley's paradox} (Lindley, 1957).
\end{itemize}
\end{itemize}
\end{frame}
\section{References}
\begin{frame}
  \frametitle{References}
  \begin{itemize}
\item  Basu, D. (1975). Statistical information and likelihood (with discussion). {\em Sankhy\={a}\/}~{\em 37\/}(1), 1--71.
\item Besag, J. and P.~Clifford (1989). Generalized {Monte Carlo} significance tests. {\em Biometrika\/} {\em 76\/}(4), 633--642.
\item Birnbaum, A. (1962). On the foundations of statistical inference. {\em Journal of the American Statistical Association\/}~{\em 57}, 269--306.
\item Birnbaum, A. (1972). More concepts of statistical evidence. {\em Journal of the American Statistical Association\/}~{\em 67}, 858--861.
\item Casella, G. and R.L.~Berger (2002). {\em Statistical Inference\/} (2nd ed.). Pacific Grove, CA, USA: Duxbury.
\item Cox, D.R. and D.V.~Hinkley (1974). {\em Theoretical Statistics}. London, UK: Chapman and Hall.
\item Dawid, A.P. (1977). Conformity of inference patterns. In J.R.~Barra et~al. (Eds.), {\em Recent Developments in Statistcs}. pp.\ 245--256. Amsterdam, The Netherlands: North-Holland Publishing Company.
\end{itemize}
\end{frame}
\begin{frame}
  \frametitle{References continued}
  \begin{itemize}
  \item  DiCiccio, T.J. and B.~Efron (1996). Bootstrap confidence intervals {\em Statistical Science\/}~{\em 11\/}(3), 189--228.
  \item Lindley, D.V. (1957). A statistical paradox. {\em Biometrika\/}~{\em 44}, 187--192.
    \item Robert, C.P. (2007). {\em The Bayesian Choice: From Decision-Theoretic Foundations to
        Computational Implementation}. New York, USA: Springer.
    \item Rubin, D.B. (1984). Bayesianly justifiable and relevant frequency calculations for the applied statistician. {\em The Annals of Statistics\/}~{\em 12\/}(4), 1151--1172.
    \item Savage, L.J. et~al. (1962). {\em The Foundations of Statistical Inference}. London, UK: Methuen.
    \item Smith, J.Q. (2010). {\em Bayesian Decision Analysis: Principle and Practice}. Cambridge, UK: Cambridge University Press.
      \item Wilks, S.S. (1938). The large-sample distribution of the likelihood ratio for testing composite hypotheses. {\em The Annals of Mathematical Statistics\/}~{\em 9}(1), 60--62.
    \end{itemize}
\end{frame}
\end{document}






