\documentclass{beamer}
\usepackage{hyperref}
% \usetheme{default}
\usetheme[secheader]{Boadilla}
%\usetheme{Hannover}
\usepackage[all]{xy}
\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage{latexsym, epsfig, subfigure , color, amsmath, bbm}
\DeclareMathOperator*{\argmin}{arg\,min}
\setbeamertemplate{blocks}[rounded][shadow]
%\setbeamercovered{transparent}
%\setbeamertemplate{navigation symbols}{}
\newcommand{\omalpha}{\ensuremath{(1 - \alpha)}\,}
\newcommand{\link}[1]{\underline{\color{red}#1}}
\newcommand{\hilite}[1]{{\color{red}#1}}
\newcommand{\Ev}{\ensuremath{\mbox{Ev}}}
\newcommand{\boost}{\vphantom{\big(}}
\newenvironment{variableblock}[3]{%
  \setbeamercolor{block body}{#2}
  \setbeamercolor{block title}{#3}
  \begin{block}{#1}}{\end{block}}
\definecolor{light-blue}{rgb}{0.29, 0.59, 0.82}
\title{Statistical Inference \\ Lecture Seven}
\subtitle{\href{https://people.bath.ac.uk/masss/APTS/2022-23/LectureSeven.pdf}{https://people.bath.ac.uk/masss/APTS/2022-23/LectureSeven.pdf}}
\author{Simon Shaw}
\institute{University of Bath}
\date{APTS, 13-16 December 2022}

\begin{document}
\begin{frame}
\titlepage
\end{frame}
\section{Confidence sets and p-values}
\subsection{Previously in Lecture Six}
\begin{frame}
  \begin{itemize}
\item A $p$-value \textcolor{red}{$p(X)$} is a \textcolor{blue}{statistic} satisfying, for every $\alpha \in [0, 1]$, \textcolor{red}{$\mathbb{P}(p(X) \leq \alpha \, | \, \theta) \leq \alpha$}. It is \textcolor{blue}{super-uniform}.
\end{itemize}
\textcolor{blue}{\textbf{Coming up in Lecture Seven}}
\begin{itemize}
\item Let \textcolor{red}{$t : \mathcal{X} \rightarrow \mathbb{R}$} be a statistic. For each \textcolor{red}{$x \in \mathcal{X}$}  and \textcolor{red}{$\theta_{0} \in \Theta$} define
\begin{eqnarray*}
\textcolor{red}{p_{t}(x; \theta_{0}) \ := \ \mathbb{P}(t(X) \geq t(x) \, | \, \theta_{0})}.
\end{eqnarray*}
Then \textcolor{red}{$p_{t}$} is a family of \textcolor{blue}{significance procedures}. \item For any finite sequence of scalar exchangeable random variables \textcolor{red}{$X_{0}, X_{1}, \ldots, X_{m}$}, then if $R$ is the \textcolor{blue}{rank} of \textcolor{red}{$X_{0}$} in the sequence then \textcolor{red}{$R$} has a \textcolor{blue}{discrete uniform distribution} on the integers \textcolor{red}{$\{0, 1, \ldots, m\}$}, and \textcolor{red}{$(R + 1)/(m + 1)$} has a \textcolor{blue}{super-uniform} distribution.
\item We utilise this result to compute the \textcolor{blue}{$p$-value} \textcolor{red}{$p_{t}(x; \theta_{0})$} corresponding to the test statistic \textcolor{red}{$t(X)$} at \textcolor{red}{$\theta_{0}$}.
\item We'll briefly look at Bayesian hypothesis testing and Lindley's Paradox.
\end{itemize}
  \end{frame} 
\subsection{Families of significance procedures}
\begin{frame}
\frametitle{Families of significance procedures}
\begin{itemize}
\item We now consider a very \textcolor{blue}{general} way to construct a family of significance procedures.
\item We will then show how to use \textcolor{blue}{simulation} to compute the family.
\end{itemize}
\begin{block}{Theorem}
Let \textcolor{red}{$t : \mathcal{X} \rightarrow \mathbb{R}$} be a statistic. For each \textcolor{red}{$x \in \mathcal{X}$}  and \textcolor{red}{$\theta_{0} \in \Theta$} define
\begin{eqnarray*}
\textcolor{red}{p_{t}(x; \theta_{0}) \ := \ \mathbb{P}(t(X) \geq t(x) \, | \, \theta_{0})}.
\end{eqnarray*}
Then \textcolor{red}{$p_{t}$} is a family of \textcolor{blue}{significance procedures}. If the distribution function of \textcolor{red}{$t(X)$} is \textcolor{blue}{continuous}, then \textcolor{red}{$p_{t}$} is \textcolor{blue}{exact}.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof (Casella and Berger, 2002)}
\begin{itemize}
\item Now 
\begin{eqnarray*}
p_{t}(x; \theta_{0}) \ = \ \mathbb{P}(t(X) \geq t(x) \, | \, \theta_{0}) \ = \ \mathbb{P}(-t(X) \leq -t(x)  \, | \, \theta_{0}).
\end{eqnarray*}
\item Let \textcolor{red}{$F$} denote the distribution function of \textcolor{red}{$Y(X) = -t(X)$} then \textcolor{red}{$p_{t}(x; \theta_{0}) = F(-t(x) \, | \, \theta_{0})$}. 
\item Assume that \textcolor{red}{$t(X)$} is \textcolor{blue}{continuous} so that $Y(X) = -t(X)$ is continuous. Using the \textcolor{blue}{Probability Integral Transform},
\begin{eqnarray*}
\textcolor{red}{\mathbb{P}(p_{t}(X; \theta_{0}) \leq \alpha \, | \, \theta_{0})} & = & \mathbb{P}(F(Y) \leq \alpha \, | \, \theta_{0}) \\ & = & \mathbb{P}(Y \leq F^{-1}(\alpha) \, | \, \theta_{0}) = F(F^{-1}(\alpha)) \textcolor{red}{= \alpha}.
\end{eqnarray*}
Hence, \textcolor{red}{$p_{t}$} is \textcolor{blue}{uniform} under \textcolor{red}{$\theta_{0}$}. 
\item It $t(X)$ is not continuous then, via the Probability Integral Transform, \textcolor{red}{$\mathbb{P}(F(Y) \leq \alpha \, | \, \theta_{0}) \leq \alpha$} and so  \textcolor{red}{$p_{t}(X; \theta_{0})$} is \textcolor{blue}{super-uniform} under \textcolor{red}{$\theta_{0}$}. \hfill $\Box$
\end{itemize} 
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item  So there is a family of significance procedures for \textcolor{blue}{each} possible function \textcolor{red}{$t : \mathcal{X} \rightarrow \mathbb{R}$}. 
\item Clearly only a tiny fraction of these can be useful functions, and the rest must be useless. 
\item Some, like \textcolor{red}{$t(x) = c$} for some constant $c$, are always useless. Others, like \textcolor{red}{$t(x) = \sin(x)$} might sometimes be a little bit useful, while others, like \textcolor{red}{$t(x) = \sum_{i} x_{i}$} might be quite useful - but it all depends on the circumstances. 
\item Some \textcolor{blue}{additional criteria} are required to separate out \textcolor{blue}{good} from \textcolor{blue}{poor} choices of the test statistic $t$, when using the construction in the theorem. 
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle{\ }
The most pertinent criterion is:
\begin{itemize}
\item Select a test statistic for which \textcolor{red}{$t(X)$} which will tend to be \textcolor{blue}{larger} for decision-relevant \textcolor{blue}{departures} from \textcolor{red}{$\theta_{0}$}. 
\end{itemize}
\begin{block}{Example}
  For the likelihood ratio, $\lambda(x)$, small observed values of $\lambda(x)$ support departures from $\theta_{0}$. Thus, \textcolor{red}{$t(X) = -2\log \lambda(X)$}, is a test statistic for which large values support departures from $\theta_{0}$.
  \end{block}
\begin{itemize}  
\item Large values of $t(X)$ will correspond to \textcolor{blue}{small} values of the \textcolor{blue}{$p$-value}, supporting the hypothesis that $H_{1}$ is true. 
\item This criterion ensures that \textcolor{red}{$p_{t}(X; \theta_{0})$} will tend to be \textcolor{blue}{smaller} under decision-relevant departures from $\theta_{0}$; small $p$-values are more interesting, precisely because \textcolor{blue}{significance procedures} are \textcolor{blue}{super-uniform} under $\theta_{0}$.
\end{itemize}
\end{frame} 
\subsection{Computing p-values}
\begin{frame}
\frametitle{Computing p-values}
Only in very special cases will it be possible to find a \textcolor{blue}{closed-form expression} for \textcolor{red}{$p_{t}$} from which we can compute the \textcolor{blue}{$p$-value} \textcolor{red}{$p_{t}(x; \theta_{0})$}.
\begin{block}{Theorem (Adapted from Besag and Clifford, 1989)}\label{theo:besag} 
For any finite sequence of scalar random variables \textcolor{red}{$X_{0}, X_{1}, \ldots, X_{m}$}, define the \textcolor{blue}{rank} of \textcolor{red}{$X_{0}$} in the sequence as
\begin{eqnarray*}
R & := & \sum_{i=1}^{m} \mathbbm{1}_{\{X_{i} \leq X_{0}\}}.
\end{eqnarray*}
If \textcolor{red}{$X_{0}, X_{1}, \ldots, X_{m}$} are \textcolor{blue}{exchangeable}\footnote{If $X_{0}, X_{1}, \ldots, X_{m}$ are exchangeable then their joint density function satisfies
$f(x_{0}, \ldots, x_{m}) = f(x_{\pi(0)}, \ldots, x_{\pi(m)})$ for all permutations $\pi$ defined on the set $\{0, \ldots, m\}$.} then \textcolor{red}{$R$} has a \textcolor{blue}{discrete uniform distribution} on the integers \textcolor{red}{$\{0, 1, \ldots, m\}$}, and \textcolor{red}{$(R + 1)/(m + 1)$} has a \textcolor{blue}{super-uniform} distribution.
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof}
By exchangeability, \textcolor{red}{$X_{0}$} has the \textcolor{blue}{same probability} of having rank \textcolor{red}{$r$} as any of the other \textcolor{red}{$X_{i}$}s, for \textcolor{blue}{any} \textcolor{red}{$r$}, and therefore
\begin{eqnarray*}
\mathbb{P}(R = r) & = & \frac{1}{m+1}
\end{eqnarray*}
for $r \in \{0, 1, \ldots, m\}$ and zero otherwise, proving the first claim. For the second claim,
$$\mathbb{P}\left(\frac{R+1}{m+1} \leq u\right) \ = \ \mathbb{P}(R+1 \leq u(m+1)) \ = \ \mathbb{P}(R +1 \leq \lfloor u(m+1) \rfloor)$$
since \textcolor{red}{$R$} is an \textcolor{blue}{integer} and \textcolor{red}{$\lfloor x \rfloor$} denotes the \textcolor{blue}{largest integer no larger} than \textcolor{red}{$x$}. 
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{block}{Proof continued}
Hence,
\begin{eqnarray}
\mathbb{P}\left(\frac{R+1}{m+1} \leq u\right) & = & \sum_{r=0}^{\lfloor u(m+1) \rfloor -1} \textcolor{red}{\mathbb{P}(R = r)} \label{eq:besag2} \\
& = & \sum_{r=0}^{\lfloor u(m+1) \rfloor -1} \textcolor{red}{\frac{1}{m+1}} \label{eq:besag3} \\
& = & \frac{\lfloor u(m+1) \rfloor}{m+1} \leq u, \nonumber
\end{eqnarray}
as required where equation (\ref{eq:besag3}) follows from  (\ref{eq:besag2}) by \textcolor{blue}{exchangeability}. \hfill $\Box$
\end{block}
\end{frame}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item We utilise this result to compute the \textcolor{blue}{$p$-value} \textcolor{red}{$p_{t}(x; \theta_{0})$} corresponding to the test statistic \textcolor{red}{$t(X)$} at \textcolor{red}{$\theta_{0}$}.
\item Fix the test statistic \textcolor{red}{$t(x)$} and define \textcolor{red}{$T_{i} = t(X_{i})$} where \textcolor{red}{$X_{1}, \ldots, X_{m}$ }are independent and identically distributed random variables with density \textcolor{red}{$f_{X}(\cdot \, | \, \theta_{0})$}.
\item Typically, we may have to use \textcolor{blue}{simulation} to obtain the sample and we'll need to specify \textcolor{red}{$\theta_{0}$} for this. 
\item Notice that $t(X), T_{1}, \ldots, T_{m}$ are exchangeable and thus \textcolor{red}{$-t(X), -T_{1}, \ldots, -T_{m}$} are \textcolor{blue}{exchangeable}.
\item Let 
\begin{eqnarray*}
R_{t}(x; \theta_{0}) \ := \ \sum_{i=1}^{m}  \mathbbm{1}_{\{-T_{i} \leq -t(x)\}} \ = \ \sum_{i=1}^{m}  \mathbbm{1}_{\{T_{i} \geq t(x)\}},
\end{eqnarray*}
then the previous theorem implies that
\begin{eqnarray*}
\textcolor{red}{P_{t}(x; \theta_{0}) \ := \ \frac{R_{t}(x; \theta_{0}) + 1}{m+1}}
\end{eqnarray*}
has a \textcolor{blue}{super-uniform} distribution under \textcolor{red}{$X \sim f_{X}(\cdot \, | \, \theta_{0})$}.
\end{itemize}
\end{frame}
\subsection{Computing $p$-values}
\begin{frame}
%\frametitle{\ }
\begin{itemize}
\item Note that \textcolor{red}{$\mathbb{P}(T \geq t(x) \, | \,\theta_{0}) = \mathbb{E}(\mathbbm{1}_{\{T \geq t(x)\}})$}.
\item Hence, the \textcolor{blue}{Weak Law of Large Numbers (WLLN)} implies that
\begin{eqnarray*}
\lim_{m \rightarrow \infty} P_{t}(x; \theta_{0}) & = & \lim_{m \rightarrow \infty} \frac{R_{t}(x; \theta_{0}) + 1}{m+1} \\
& = & \lim_{m \rightarrow \infty} \frac{R_{t}(x; \theta_{0}) }{m} \\
& = & \textcolor{red}{\lim_{m \rightarrow \infty} \frac{\sum_{i=1}^{m}  \mathbbm{1}_{\{T_{i} \geq t(x)\}}}{m}} \\
& = & \mathbb{P}(T \geq t(x) \, | \,\theta_{0}) \ = \ p_{t}(x; \theta_{0}).
\end{eqnarray*}
\item Therefore, not only is \textcolor{red}{$P_{t}(x; \theta_{0})$} \textcolor{blue}{super-uniform} under \textcolor{red}{$\theta_{0}$}, so that $P_{t}$ is a family of significance procedures for every $m$, but the \textcolor{blue}{limiting value} of \textcolor{red}{$P_{t}(x; \theta_{0})$} as $m$ becomes large is \textcolor{red}{$p_{t}(x; \theta_{0})$}.
\item In summary, if you can \textcolor{blue}{simulate} from your model under \textcolor{red}{$\theta_{0}$} then you can produce a $p$-value for \textcolor{blue}{any test statistic} \textcolor{red}{$t$}, namely \textcolor{red}{$P_{t}(x; \theta_{0})$}, and if you can simulate cheaply, so that the number of simulations m is large, then \textcolor{red}{$P_{t}(x; \theta_{0}) \approx p_{t}(x; \theta_{0})$}.
\end{itemize}
\end{frame}
\begin{frame}
%\frametitle}\ }
\begin{itemize}
\item However, this simulation-based approach is not well-adapted to constructing \textcolor{blue}{confidence sets}.
\item Let \textcolor{red}{$C_{t}$} be the family of \textcolor{blue}{confidence procedures} induced by \textcolor{red}{$p_{t}$} using \textcolor{blue}{duality}.
\item With \textcolor{blue}{one set} of $m$ simulations, we can answer "Is \textcolor{red}{$\theta_{0} \in C_{t}(x; \alpha)$}?"
\begin{itemize}
\item These simulations give a value \textcolor{red}{$P_{t}(x; \theta_{0})$} which is either larger or not larger than \textcolor{red}{$\alpha$}.
\item If \textcolor{red}{$P_{t}(x; \theta_{0}) > \alpha$} then \textcolor{red}{$\theta_{0} \in C_{t}(x; \alpha)$}, and otherwise it is not.
\end{itemize}
\item  However, this is \textcolor{blue}{not an effective way} to enumerate all of the points in \textcolor{red}{$C_{t}(x; \alpha)$} since we would need to do $m$ \textcolor{blue}{simulations} for \textcolor{blue}{each point} in \textcolor{red}{$\Theta$}.
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Interpretations}
\begin{itemize}
\item It is a very common observation, made repeatedly over the last 50 years see, for example, Rubin (1984), that clients think more like Bayesians than classicists. 
\item For example, \textcolor{red}{$\mathbb{P}(\theta \in C(X; \alpha) \, | \, \theta) \geq 1 - \alpha$} is often interpreted as a probability over \textcolor{red}{$\theta$} for the \textcolor{blue}{observed} \textcolor{red}{$C(x; \alpha)$}.
\item Classical  statisticians thus have to wrestle with the issue that their clients will likely \textcolor{blue}{misinterpret} their results.
\item We will now briefly look at Bayesian approaches to hypothesis testing.
\item In this approach, we can calculate the posterior probability of each hypothesis.
\end{itemize}
\end{frame}
\subsection{Bayesian approaches to hypothesis testing}
\begin{frame}
\begin{itemize}
\item Consider a point-null hypothesis \textcolor{red}{$H_{0}: \theta = \theta_{0}$} versus \textcolor{red}{$H_{1}: \theta \neq \theta_{0}$}.
\item A possible prior is a mixture of a \textcolor{blue}{point mass} on $\theta_{0}$ and a \textcolor{blue}{distribution}, $\pi_{1}(\theta)$, under $H_{1}$:
\begin{eqnarray*}
\textcolor{red}{\pi(\theta) \ = \ p_{0}\mathbb{I}_{\{\theta = \theta_{0}\}}  + (1-p_{0})\pi_{1}(\theta)}
\end{eqnarray*}
where \textcolor{red}{$p_{0} = \mathbb{P}(\theta = \theta_{0})$}.
\item If $f_{X}(x \, | \, \theta)$ is the data generating model then the \textcolor{blue}{posterior probability} of $\theta = \theta_{0}$ is
\begin{eqnarray*}
\mathbb{P}(\theta = \theta_{0} \, | \, X) \ = \ \frac{p_{0}f_{X}(x \, | \, \theta_{0})}{\int f_{X}(x \, | \, \theta)\pi(\theta) d\theta} \ = \ \frac{p_{0}f_{X}(x \, | \, \theta_{0})}{p_{0}\textcolor{red}{f_{X}(x \, | \, \theta_{0})} + (1-p_{0})\textcolor{red}{f_{1}(x)}}
\end{eqnarray*}
where \textcolor{red}{$f_{1}(x)$} is the marginal distribution under $H_{1}$,
\begin{eqnarray*}
f_{1}(x) & = & \int_{\Theta_{1}} f_{X}(x \, | \, \theta)\pi_{1}(\theta) d\theta
\end{eqnarray*}
\item Thus, \textcolor{red}{$\mathbb{P}(\theta = \theta_{0} \, | \, X) = (1+y)^{-1}$} where
\begin{eqnarray*}
\textcolor{red}{y \ = \ \frac{1-p_{0}}{p_{0}} \frac{f_{1}(x)}{f_{X}(x \, | \, \theta_{0})}}. 
\end{eqnarray*}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Example: normal model for $H_{0} : \theta = 0$}
\begin{itemize}
\item Let  \textcolor{red}{$\theta_{0} = 0$} and suppose that \textcolor{red}{$X \, | \, \theta \sim N(\theta, \sigma^{2})$} for $\sigma^{2}$ known. 
\item For the prior under \textcolor{red}{$H_{1}: \theta \neq 0$} we assert \textcolor{red}{$\theta \sim N(0, \sigma_{0}^{2})$} where $\sigma_{0}^{2}$ is known.
\item Thus,
\begin{eqnarray*}
f_{X}(x \, | \, \theta = 0) & = & \frac{1}{\sqrt{2\pi}\sigma} \exp \left\{-\frac{1}{2\sigma^{2}} x^{2}\right\}, \\ 
f_{1}(x) & = & \int_{-\infty}^{\infty} f_{X}(x \, | \, \theta)\pi_{1}(\theta) d\theta \\
& = &  \int_{-\infty}^{\infty} \frac{1}{2\pi\sigma\sigma_{0}} \exp \left\{-\frac{1}{2\sigma^{2}} (x-\theta)^{2}-\frac{1}{2\sigma_{0}^{2}} \theta^{2}\right\} d\theta \\
& = & \frac{(\sigma^{2}+\sigma_{0}^{2})^{-\frac{1}{2}}}{\sqrt{2\pi}} \left\{-\frac{x^{2}}{2(\sigma^{2}+\sigma_{0}^{2})} \right\} 
\end{eqnarray*}
so that \textcolor{red}{$f_{1}(x)$} is the pdf of \textcolor{red}{$N(0, \sigma^{2}+\sigma_{0}^{2})$}.
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Example: normal model for $H_{0} : \theta = 0$}
\begin{itemize}
\item Hence, \textcolor{red}{$\mathbb{P}(\theta = 0 \, | \, X = x) = (1+y)^{-1}$} where
\begin{eqnarray*}
y & = & \frac{1-p_{0}}{p_{0}} \textcolor{red}{\frac{f_{1}(x)}{f_{X}(x \, | \, \theta = 0)}} \\
& = & \left(\frac{1-p_{0}}{p_{0}}\right)\left(\frac{\sigma^{2}}{\sigma^{2} + \sigma_{0}^{2}}\right)^{\frac{1}{2}} \exp\left\{ -\frac{1}{2}\left(\frac{1}{\sigma^{2}+\sigma_{0}^{2}} - \frac{1}{\sigma^{2}}\right) x^{2}\right\} \\
& = & \left(\frac{1-p_{0}}{p_{0}}\right)\left(\frac{\sigma^{2}}{\sigma^{2} + \textcolor{red}{\sigma_{0}^{2}}}\right)^{\frac{1}{2}} \exp\left\{\frac{\textcolor{red}{\sigma_{0}^{2}}x^{2}}{2(\sigma^{2}+\textcolor{red}{\sigma_{0}^{2}})\sigma^{2}}\right\} 
\end{eqnarray*}
\item A disperse prior for \textcolor{red}{$H_{1}: \theta \neq 0$} is sometimes proposed and this can be achieved by increasing the prior variance \textcolor{red}{$\sigma_{0}^{2}$}. 
    \item If \textcolor{red}{$\sigma_{0}^{2} \rightarrow \infty$} then \textcolor{red}{$y \rightarrow 0$} and \textcolor{red}{$\mathbb{P}(\theta = 0 \, | \, X = x) \rightarrow 1$} for \textcolor{blue}{all} \textcolor{red}{$x$}. This may be an \textcolor{blue}{issue} with using \textcolor{blue}{improper priors}: a proper prior has $\sigma_{0}^{2}$ finite.
\item Note that $y$ increases in $|x|$ and so $\mathbb{P}(\theta = 0 \, | \, X = x)$ decreases.
\item With a proper prior, as \textcolor{red}{$|x| \rightarrow \infty$}, \textcolor{red}{$y \rightarrow \infty$} and \textcolor{red}{$\mathbb{P}(\theta = 0 \, | \, X = x) \rightarrow 0$}. The Bayesian analysis behaves reasonably.
\end{itemize}
\end{frame}
\begin{frame}
\begin{itemize}
\item Now consider taking $n$ iid observations and consider the posterior probability given $\overline{x}$. 
\item Notice that, as \textcolor{red}{$\overline{X} \, | \, \theta \sim N(\theta, \sigma^{2}/n)$}, our calculations will take the same form as previously but with $x$ replaced by $\overline{x}$ and $\sigma^{2}$ by $\sigma^{2}/n$.
\item Thus,  \textcolor{red}{$\mathbb{P}(\theta = 0 \, | \, \overline{X} = \overline{x}) = (1+y_{n})^{-1}$} where
\begin{eqnarray*}
y_{n} & = & \left(\frac{1-p_{0}}{p_{0}}\right)\left(\frac{\sigma^{2}}{\sigma^{2} + n\sigma_{0}^{2}}\right)^{\frac{1}{2}} \exp\left\{\frac{n^{2}\sigma_{0}^{2}\overline{x}^{2}}{2(\sigma^{2}+ n\sigma_{0}^{2})\sigma^{2}}\right\} \\
& = & \left(\frac{1-p_{0}}{p_{0}}\right)\left(\frac{\sigma^{2}}{\sigma^{2} + n\sigma_{0}^{2}}\right)^{\frac{1}{2}} \exp\left\{\frac{n\sigma_{0}^{2}}{2(\sigma^{2}+ n\sigma_{0}^{2})}\textcolor{red}{z^{2}}\right\}
\end{eqnarray*}
and \textcolor{red}{$z = \sqrt{n}|\overline{x}|/\sigma$}.
\item Note that if $H_{0}$ is true then \textcolor{red}{$\sqrt{n}\overline{X}/\sigma \sim N(0, 1)$} so \textcolor{red}{$Z^{2} \sim \chi^{2}_{1}$}.
\item Suppose that \textcolor{red}{$z = \sqrt{n}|\overline{x}|/\sigma$} is \textcolor{blue}{fixed} as we increase \textcolor{red}{$n$}. Then \textcolor{red}{$y_{n} \rightarrow 0$} and hence \textcolor{red}{$\mathbb{P}(\theta = 0 \, | \, \overline{X} = \overline{x}) \rightarrow 1$}.
\item The Bayesian model favours $H_{0}$ over $H_{1}$.
\end{itemize}
\end{frame}
\begin{frame}
\begin{itemize}
\item Now let's consider the \textcolor{blue}{classical approach} to this problem using a \textcolor{blue}{$p$-value}.
\item Consider the test statistic \textcolor{red}{$|\overline{X}|$} which will be \textcolor{blue}{large} for departures from \textcolor{red}{$H_{0}: \theta_{0} = 0$}. We have
\begin{eqnarray*}
p(|\overline{x}|; 0) & = & \mathbb{P}(|\overline{X}| \geq |\overline{x}| \, | \, \theta = 0) \\
& = & \mathbb{P}(\sqrt{n}|\overline{X}|/\sigma \geq \textcolor{red}{z} \, | \, \theta = 0).
\end{eqnarray*}
\item Now, under $H_{0}$, \textcolor{red}{$\sqrt{n}\overline{X}/\sigma \sim N(0, 1)$}. If \textcolor{red}{$z = \sqrt{n}|\overline{x}|/\sigma$} is \textcolor{blue}{fixed} for all $n$ then the $p$-value is \textcolor{blue}{fixed} for all $n$.
\item Thus, if \textcolor{red}{$\alpha \geq p(|\overline{x}|; 0)$} we \textcolor{blue}{reject} $H_{0}$ for all values of $n$ at significance level $\alpha$.
\item This is an illustration of what is termed \textcolor{blue}{Lindley's paradox} (Lindley, 1957).
\end{itemize}
\end{frame} 
\begin{frame}
\frametitle{Lindley's paradox}
The main idea of this seeming paradox can be expressed as follows.
\begin{itemize}
\item For a normal model $N(\theta, \sigma^{2})$ with known variance $\sigma^{2}$, consider the hypothesis test $H_{0}: \theta = \theta_{0}$ versus $H_{1}: \theta \neq \theta_{0}$.
\item Assume \textcolor{red}{$\mathbb{P}(\theta \in H_{0}) > 0$} and any regular prior on $\{\theta \neq \theta_{0}\}$. Then for \textcolor{blue}{any} \textcolor{red}{$\alpha \in [0, 1]$} we can find a \textcolor{blue}{sample size} \textcolor{red}{$n(\alpha)$} and iid \textcolor{blue}{data} $x_{1}, \dots, x_{n}$ such that:
\begin{enumerate}
\item The sample mean $\overline{x}$ is significantly different from $H_{0}$ at level $\alpha$.
\item The posterior probability that $\theta = \theta_{0}$ is greater that $1-\alpha$.
\end{enumerate}
\item In our example, if we set \textcolor{red}{$\sigma^{2} = \sigma_{0}^{2} = 1$}, \textcolor{red}{$n = 16818$}, and \textcolor{red}{$\overline{x} = 1.96(16818)^{-\frac{1}{2}} = 0.015$} then \textcolor{red}{$z = 1.96$} and \textcolor{red}{$\mathbb{P}(\theta = 0 \, | \, \overline{X} = \overline{x}) = 0.95$}
\item The reasoning for this seeming paradox is that the classical and Bayesian approaches are asking \textcolor{blue}{different questions}.
\end{itemize}
\end{frame}
\subsection{Concluding remarks}
\begin{frame}
\frametitle{Concluding remarks: understanding the problem}
\begin{itemize}
\item A $p$-value \textcolor{red}{$p(x; \theta_{0})$} refers only to \textcolor{red}{$\theta_{0}$}, making \textcolor{blue}{no reference} at all to other hypotheses about \textcolor{red}{$\theta$}.
\begin{itemize}
\item A $p$-value can be viewed as \textcolor{blue}{measuring the fit} of a model, that under $H_{0}$, to the observed data. 
\item If I reject $H_{0}$ using a $p$-value then $H_{0}$ is a \textcolor{blue}{poor explanation} for the observation.
\item However, a \textcolor{blue}{large} $p$-value indicates only that the data is \textcolor{blue}{not unusual} under the model but it does not imply that the model is correct. 
\item For example, there may be many other models defined by other hypotheses which may be exhibit \textcolor{blue}{greater consistency} with the observed data.
\end{itemize}
\item A posterior probability \textcolor{red}{$\pi(\theta_{0} \, | \, x)$} \textcolor{blue}{contrasts} \textcolor{red}{$\theta_{0}$} with the other values in $\Theta$ which \textcolor{red}{$\theta$} might have taken. 
\begin{itemize}
\item If I favour $H_{0}$ then $H_{0}$ is a better explanation for the data $x$ than $H_{1}$.
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\begin{itemize}
\item Wasserstein and Lazar (2016) is a statement from the American Statistical Association (ASA) on statistical significance and $p$-values.
\item The statement gives six principles for the correct use and interpretation of $p$-values.
\item These principles, in particular Principles 3 and 4, reflect values that should be at the heart of any work that we do.
\end{itemize}
\textbf{Scientific conclusions and business or policy decisions should not be based only on whether a p-value passes a specific threshold.}
\begin{itemize}
\item Practices that reduce data analysis or scientific inference to mechanical ``bright-line'' rules (such as ``$p < 0.05$'') for justifying scientific claims or conclusions can lead to \textcolor{blue}{erroneous beliefs and poor decision making}.
\item Researchers should bring many contextual factors into play to derive scientific inferences, including the design of a study, the quality of the measurements, the external evidence for the phenomenon under study, and the \textcolor{blue}{validity of assumptions} that underlie the data analysis.
\end{itemize}
\end{frame}
\begin{frame}
\textbf{Proper inference requires full reporting and transparency.}
\begin{itemize}
\item Whenever a researcher chooses what to present based on statistical results, valid interpretation of those results is \textcolor{blue}{severely compromised} if the reader is \textcolor{blue}{not informed} of the choice and its basis.
\item Researchers should \textcolor{blue}{disclose} the number of hypotheses explored during the study, all data collection decisions, all statistical analyses conducted, and all p-values computed.
\end{itemize}
\end{frame}
\end{document}



\begin{frame}
\begin{itemize}
\item This difference in approach can be seen if I consider what I need to compute the posterior probability when I have a $p$-value.
\item Suppose I observe \textcolor{red}{$p(x; H_{0}) \leq u$} then:
\begin{eqnarray*}
\mathbb{P}(H_{0} \, | \, p(x; H_{0}) \leq u) & = & \frac{\mathbb{P}(p(X; H_{0}) \leq u \, | \, H_{0})\mathbb{P}(H_{0})}{\mathbb{P}(p(X; H_{0}) \leq u)} \\
& \leq & \frac{u \mathbb{P}(H_{0})}{\mathbb{P}(p(X; H_{0}) \leq u)} 
\end{eqnarray*}
since the $p$-value is super-uniform.
\item Now,
\begin{eqnarray*}
\mathbb{P}(p(X; H_{0}) \leq u) & = & \sum_{i=0}^{1} \mathbb{P}(p(X; H_{0}) \leq u \, | \, H_{i})\mathbb{P}(H_{i})
\end{eqnarray*}
and so I need to know the distribution of the $p$-value under $H_{1}$ to compute \textcolor{red}{$\mathbb{P}(p(X; H_{0}) \leq u \, | \, H_{1})$}.
\end{itemize}
\end{frame}
\section{References}
\begin{frame}
  \frametitle{References}
  \begin{itemize}
\item  Basu, D. (1975). Statistical information and likelihood (with discussion). {\em Sankhy\={a}\/}~{\em 37\/}(1), 1--71.
\item Besag, J. and P.~Clifford (1989). Generalized {Monte Carlo} significance tests. {\em Biometrika\/} {\em 76\/}(4), 633--642.
\item Birnbaum, A. (1962). On the foundations of statistical inference. {\em Journal of the American Statistical Association\/}~{\em 57}, 269--306.
\item Birnbaum, A. (1972). More concepts of statistical evidence. {\em Journal of the American Statistical Association\/}~{\em 67}, 858--861.
\item Casella, G. and R.L.~Berger (2002). {\em Statistical Inference\/} (2nd ed.). Pacific Grove, CA, USA: Duxbury.
\item Cox, D.R. and D.V.~Hinkley (1974). {\em Theoretical Statistics}. London, UK: Chapman and Hall.
\item Dawid, A.P. (1977). Conformity of inference patterns. In J.R.~Barra et~al. (Eds.), {\em Recent Developments in Statistcs}. pp.\ 245--256. Amsterdam, The Netherlands: North-Holland Publishing Company.
\end{itemize}
\end{frame}
\begin{frame}
  \frametitle{References continued}
  \begin{itemize}
  \item  DiCiccio, T.J. and B.~Efron (1996). Bootstrap confidence intervals {\em Statistical Science\/}~{\em 11\/}(3), 189--228.
  \item Lindley, D.V. (1957). A statistical paradox. {\em Biometrika\/}~{\em 44}, 187--192.
    \item Robert, C.P. (2007). {\em The Bayesian Choice: From Decision-Theoretic Foundations to
        Computational Implementation}. New York, USA: Springer.
    \item Rubin, D.B. (1984). Bayesianly justifiable and relevant frequency calculations for the applied statistician. {\em The Annals of Statistics\/}~{\em 12\/}(4), 1151--1172.
    \item Savage, L.J. et~al. (1962). {\em The Foundations of Statistical Inference}. London, UK: Methuen.
    \item Smith, J.Q. (2010). {\em Bayesian Decision Analysis: Principle and Practice}. Cambridge, UK: Cambridge University Press.
      \item Wilks, S.S. (1938). The large-sample distribution of the likelihood ratio for testing composite hypotheses. {\em The Annals of Mathematical Statistics\/}~{\em 9}(1), 60--62.
    \end{itemize}
\end{frame}
\end{document}






