\documentclass[a4paper]{article}
\textwidth= 400pt
\oddsidemargin = 30pt
\marginparwidth = 0pt
\usepackage{latexsym, graphics, epsfig, amssymb, color, amsfonts, amsmath, framed,fancybox, subfigure, dsfont}
\usepackage{tikz, bbm}
\usepackage{titlesec}
\definecolor{shadecolor}{rgb}{0.29, 0.59, 0.82}
\definecolor{shadecolorI}{rgb}{0.94, 1.0, 1.0}
\definecolor{light-blue}{rgb}{0.29, 0.59, 0.82}
\definecolor{iceberg}{rgb}{0.44, 0.65, 0.82}
\definecolor{lightcornflowerblue}{rgb}{0.6, 0.81, 0.93}
\definecolor{skyblue}{rgb}{0.53, 0.81, 0.92}
\definecolor{airforceblue}{rgb}{0.36, 0.54, 0.66}
\definecolor{bluegray}{rgb}{0.4, 0.6, 0.8}
\definecolor{arsenic}{rgb}{0.23, 0.27, 0.29}
\definecolor{babyblue}{rgb}{0.54, 0.81, 0.94}
\titleformat{\chapter}
{\color{light-blue}\normalfont\huge\bfseries}
{\color{light-blue}\thechapter}{1em}{}
\titleformat{\section}
{\color{light-blue}\normalfont\Large\bfseries}
{\color{light-blue}\thesection}{1em}{}

\titleformat{\subsection}
{\color{light-blue}\normalfont\large\bfseries}
{\color{light-blue}\thesubsection}{1em}{}

\usepackage[colorlinks=true, linkcolor=light-blue,
            urlcolor=blue,
            citecolor=blue]{hyperref}          
\DeclareMathOperator*{\argsup}{arg\,sup}
\DeclareMathOperator*{\argmin}{arg\,min}
\textwidth= 400pt
\oddsidemargin = 30pt
\marginparwidth = 0pt
%\numberwithin{equation}{section}
\begin{document}
\title{\color{light-blue}{Solutions to APTS Assessment on Statistical Inference}}
\author{\color{light-blue}{Simon Shaw, s.shaw@bath.ac.uk} \\ \color{light-blue}{University of Bath}}
\date{\color{light-blue}{Warwick, 13-16 December 2022}}
\maketitle
\newtheorem{Theo}{Theorem}
\newtheorem{Lem}{Lemma}
\newtheorem{Cor}{Corollary}
\newtheorem{Def}{Definition}
\newtheorem{Prin}{Principle}
\newcommand{\Ev}{\ensuremath{\mbox{Ev}}}


\newtheorem{Ex}{Example}
\newcommand{\conf}{\mathop{\textrm{conf}}}
\newcommand{\Chat}{\widehat{C}}
\newcommand{\betahat}{\hat{\beta}}
\newcommand{\Ctilde}{\tilde{C}}
%\newcommand{\dtilde}{\tilde{\delta}}
\newcommand{\ftilde}{\tilde{f}}
\newcommand{\Ftilde}{\widetilde{\curlyF}}
\newcommand{\Gtilde}{\widetilde{\curlyG}}
\newcommand{\R}{\textsf{R}}
\newenvironment{literalR}{\small\verbatim}{\endverbatim}
\section*{Principles for Statistical Inference}
\begin{enumerate}
\item \textbf{Consider Birnbaum's Theorem, $(\mbox{WIP} \wedge \mbox{WCP} \, ) \leftrightarrow \mbox{SLP}$. In lectures, we showed that $(\mbox{WIP}
  \wedge \mbox{WCP} \,) \rightarrow \mbox{SLP}$ but not the converse. Hence, show that $\mbox{SLP} \rightarrow \mbox{WIP}$ and $\mbox{SLP} \rightarrow  \mbox{WCP}$.}

\ \newline
The Strong Likelihood Principle (SLP) states that if $\mathcal{E}_{1} = \{\mathcal{X}_{1}, \Theta, f_{X_{1}}(x_{1} \, | \, \theta)\}$ and $\mathcal{E}_{2} = \{\mathcal{X}_{2}, \Theta, f_{X_{2}}(x_{2} \, | \, \theta)\}$ are two experiments with the same parameter $\theta$ and if $x_{1} \in \mathcal{X}_{1}$ and $x_{2} \in \mathcal{X}_{2}$ satisfy $f_{X_{1}}(x_{1} \, | \, \theta) = c(x_{1}, x_{2})f_{X_{2}}(x_{2} \, | \, \theta)$ for some $c > 0$ for all $\theta \in \Theta$ then $Ev(\mathcal{E}_{1}, x_{1}) = Ev(\mathcal{E}_{2}, x_{2})$.
\begin{enumerate}
\item $\mbox{SLP} \rightarrow \mbox{WIP}$.  \newline
The Weak Indifference Principle (WIP) states that for the experiment $\mathcal{E} = \{\mathcal{X}, \Theta$, $f_{X}(x \, | \, \theta)\}$ if $f_{X}(x \, | \, \theta) = f_{X}(x' \, | \, \theta)$ for all $\theta \in \Theta$ then $Ev(\mathcal{E}, x) = Ev(\mathcal{E}, x')$.

\ \newline
 In the SLP, let $\mathcal{E}_{1} = \mathcal{E}_{2} = \mathcal{E}$ and suppose that $f_{X}(x \, | \, \theta) = f_{X}(x' \, | \, \theta)$ for all $\theta \in \Theta$. Hence, taking $c(x, x') = 1$, the SLP implies that $Ev(\mathcal{E}, x) = Ev(\mathcal{E}, x')$ which is the WIP. 
\item $\mbox{SLP} \rightarrow  \mbox{WCP}$. \newline
The Weak Conditionality Principle (WCP) states that if $\mathcal{E}^{*}$ is the mixture of the experiments $\mathcal{E}_{1}$ and $\mathcal{E}_{2}$ according to mixture probabilities $p_{1}, p_{2} = 1 - p_{1}$ then $Ev(\mathcal{E}^{*}, (i, x_{i})) = Ev(\mathcal{E}_{i}, x_{i})$.

\ \newline
For the mixture experiment we have $f^{*}((i, x_{i}) \, | \, \theta) = p_{i}f_{X_{i}}(x_{i} \, | \, \theta)$ for all $\theta \in \Theta$. Applying the SLP with $c((i, x_{i}), x_{i}) = p_{i}$ gives $Ev(\mathcal{E}^{*}, (i, x_{i})) = Ev(\mathcal{E}_{i}, x_{i})$ which is the WCP.
\end{enumerate}
\item \footnote{See Section 5 of Berger, J. (1985). In defense of the likelihood principle: Axiomatics and coherency. \textit{Bayesian Statistics 2} (J.M. Bernardo, M.H. DeGroot, D.V. Lindley, and A.F.M. Smith, Eds.), 33-66. North-Holland.}\textbf{Suppose that we have two discrete experiments $\mathcal{E}_{1} = \{\mathcal{X}_{1}, \Theta, f_{X_{1}}(x_{1} \, | \, \theta)\}$ and $\mathcal{E}_{2} = \{\mathcal{X}_{2}, \Theta, f_{X_{2}}(x_{2} \, | \, \theta)\}$ and that, for $x_{1}' \in \mathcal{X}_{1}$ and $x_{2}' \in \mathcal{X}_{2}$,} 
\begin{eqnarray}
f_{X_{1}}(x_{1}' \, | \, \theta) & = & cf_{X_{2}}(x_{2}' \, | \, \theta) \label{eq:slp}
\end{eqnarray}
\textbf{for all $\theta$ where $c$ is a positive constant not depending upon $\theta$ (but which may depend on $x_{1}', x_{2}'$) and $f_{X_{1}}(x_{1}' \, | \, \theta) > 0$. We wish to consider estimation of $\theta$ under a loss function $L(\theta, d)$ which is strictly convex in $d$ for each $\theta$. Thus, for all $d_{1} \neq d_{2} \in \mathcal{D}$, the decision space, and $\alpha \in (0, 1)$,}
\begin{eqnarray*}
L(\theta, \alpha d_{1} + (1-\alpha)d_{2}) & < & \alpha L(\theta, d_{1}) + (1-\alpha)L(\theta, d_{2}).
\end{eqnarray*}
\textbf{For the experiment $\mathcal{E}_{j}$, $j = 1, 2$, for the observation $x_{j}$ we will use the decision rule $\delta_{j}(x_{j})$ as our estimate of $\theta$ so that} 
\begin{eqnarray*}
\Ev(\mathcal{E}_{j}, x_{j}) & = & \delta_{j}(x_{j}).
\end{eqnarray*}
\textbf{Suppose that the inference violates the strong likelihood principle so that, whilst equation (\ref{eq:slp}) holds, $\delta_{1}(x_{1}') \neq \delta_{2}(x_{2}')$.}
\begin{enumerate}
\item \textbf{Let $\mathcal{E}^{*}$ be the mixture of the experiments $\mathcal{E}_{1}$ and $\mathcal{E}_{2}$ according to mixture probabilities $1/2$ and $1/2$. For the outcome $(j, x_{j})$ the decision rule is $\delta(j, x_{j})$. If the Weak Conditionality Principle (WCP) applies to $\mathcal{E}^{*}$ show that}
\begin{eqnarray*}
\delta(1, x_{1}') & \neq & \delta(2, x_{2}').
\end{eqnarray*}
Under the WCP, $\Ev(\mathcal{E}^{*}, (j, x_{j})) = \Ev(\mathcal{E}_{j}, x_{j})$ so that $\delta(j, x_{j}) = \delta_{j}(x_{j})$. Thus, if $\delta_{1}(x_{1}') \neq \delta_{2}(x_{2}')$ it immediately follows that $\delta(1, x_{1}') \neq \delta(2, x_{2}')$.
\item \textbf{An alternative decision rule for $\mathcal{E}^{*}$ is}
\begin{eqnarray}
\delta^{*}(j, x_{j}) & = & \left\{\begin{array}{ll} 
\frac{c}{c+1}\delta(1, x_{1}') + \frac{1}{c+1}\delta(2, x_{2}') & \mbox{if $x_{j} = x_{j}'$ for $j = 1, 2$}, \\
\delta(j, x_{j}) & \mbox{otherwise}.
\end{array} \right. \label{eq:slp1}
\end{eqnarray}
\textbf{Show that if the WCP applies to $\mathcal{E}^{*}$ then $\delta^{*}$ dominates $\delta$ so that $\delta$ is inadmissible. \newline
[Hint: First show that $R(\theta, \delta^{*}) = \frac{1}{2}\mathbb{E}[L(\theta, \delta^{*}(1, X_{1})) \, | \, \theta] + \frac{1}{2}\mathbb{E}[L(\theta, \delta^{*}(2, X_{2})) \, | \, \theta]$.]}

\ \newline
In the mixture experiment the pair $(j, x_{j})$ are random and the classical risk for $\delta^{*}$ is
\begin{eqnarray}
R(\theta, \delta^{*}) & = & \mathbb{E}[L(\theta, \delta^{*}(J, X_{J})) \, | \, \theta] \nonumber \\
& = & \sum_{j} \sum_{x_{j}} L(\theta, \delta^{*}(j, x_{j}))f^{*}((j, x_{j}) \, | \, \theta) \nonumber \\
& = & \sum_{j} \sum_{x_{j}} L(\theta, \delta^{*}(j, x_{j}))\frac{1}{2}f_{X_{j}}(x_{j} \, | \, \theta) \nonumber \\
& = & \frac{1}{2}\mathbb{E}[L(\theta, \delta^{*}(1, X_{1})) \, | \, \theta] + \frac{1}{2}\mathbb{E}[L(\theta, \delta^{*}(2, X_{2})) \, | \, \theta]. \label{eq:classrisk1}
\end{eqnarray}
In an identical fashion it follows that 
\begin{eqnarray}
R(\theta, \delta) & = & \frac{1}{2}\mathbb{E}[L(\theta, \delta(1, X_{1})) \, | \, \theta] + \frac{1}{2}\mathbb{E}[L(\theta, \delta(2, X_{2})) \, | \, \theta]. \label{eq:classrisk2}
\end{eqnarray}
Now, for each $j = 1, 2$, as $\delta^{*}(j, x_{j})  = \delta(j, x_{j})$ for all $x_{j} \neq x_{j}'$, 
\begin{eqnarray}
\lefteqn{\mathbb{E}[L(\theta, \delta^{*}(j, X_{j})) \, | \, \theta] \ = \ \sum_{x_{j}}  L(\theta, \delta^{*}(j, x_{j}))f_{X_{j}}(x_{j} \, | \, \theta)} \nonumber \\
& = & \sum_{x_{j}}  L(\theta, \delta(j, x_{j}))f_{X_{j}}(x_{j} \, | \, \theta) + \{L(\theta, \delta^{*}(j, x_{j}'))-L(\theta, \delta(j, x_{j}'))\}f_{X_{j}}(x_{j}' \, | \, \theta) \nonumber \\
& = & \mathbb{E}[L(\theta, \delta(j, X_{j})) \, | \, \theta] + \{L(\theta, \delta^{*}(j, x_{j}'))-L(\theta, \delta(j, x_{j}'))\}f_{X_{j}}(x_{j}' \, | \, \theta). \label{eq:classrisk3}
\end{eqnarray}
Substituting, for each $j$, equation (\ref{eq:classrisk3}) into (\ref{eq:classrisk1}) and using (\ref{eq:classrisk2}) gives 
\begin{eqnarray}
R(\theta, \delta^{*}) & = & R(\theta, \delta) + \frac{1}{2}\{L(\theta, \delta^{*}(1, x_{1}'))-L(\theta, \delta(1, x_{1}'))\}f_{X_{1}}(x_{1}' \, | \, \theta) + \nonumber \\
& & \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \frac{1}{2}\{L(\theta, \delta^{*}(2, x_{2}'))-L(\theta, \delta(2, x_{2}'))\}f_{X_{2}}(x_{2}' \, | \, \theta) \nonumber \\
& = & R(\theta, \delta) + \frac{1}{2}\{L(\theta, \delta^{*}(1, x_{1}'))-L(\theta, \delta(1, x_{1}'))\}f_{X_{1}}(x_{1}' \, | \, \theta) + \nonumber \\
& & \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \frac{1}{2c}\{L(\theta, \delta^{*}(2, x_{2}'))-L(\theta, \delta(2, x_{2}'))\}f_{X_{1}}(x_{1}' \, | \, \theta) \label{eq:classrisk4}
\end{eqnarray}
using equation (\ref{eq:slp}). Now, from equation (\ref{eq:slp1}), $\delta^{*}(1, x_{1}') = \delta^{*}(2, x_{2}')$ and so, for all $\theta$,  $L(\theta, \delta^{*}(1, x_{1}')) = L(\theta, \delta^{*}(2, x_{2}'))$. Hence, (\ref{eq:classrisk4}) becomes
\begin{eqnarray}
\lefteqn{R(\theta, \delta^{*}) \ = \ R(\theta, \delta)} \nonumber \\
 & &  + \frac{f_{X_{1}}(x_{1}' \, | \, \theta) }{2c}\{(c+1)L(\theta, \delta^{*}(1, x_{1}')) - cL(\theta, \delta(1, x_{1}')) - L(\theta, \delta(2, x_{2}'))\} \nonumber \\
& = & R(\theta, \delta) + \frac{(c+1)f_{X_{1}}(x_{1}' \, | \, \theta) }{2c} A(\theta) \label{eq:classrisk5}
\end{eqnarray}
where 
\begin{eqnarray}
A(\theta) & = & L(\theta, \delta^{*}(1, x_{1}')) - \frac{c}{c+1}L(\theta, \delta(1, x_{1}')) - \frac{1}{c+1}L(\theta, \delta(2, x_{2}')) \nonumber \\
& = & L\left(\theta, \frac{c}{c+1}\delta(1, x_{1}') + \frac{1}{c+1}\delta(2, x_{2}') \right) - \nonumber \\
& & \left( \frac{c}{c+1}L(\theta, \delta(1, x_{1}')) + \frac{1}{c+1}L(\theta, \delta(2, x_{2}'))\right) \nonumber \\
& < & 0 \nonumber
\end{eqnarray}
by the strict convexity of $L(\theta, d)$ in $d$ for each $\theta$ as $\delta(1, x_{1}') \neq \delta(2, x_{2}')$. Hence, using equation (\ref{eq:classrisk5}), we have for each $\theta$ that
\begin{eqnarray*}
R(\theta, \delta^{*}) & < & R(\theta, \delta)
\end{eqnarray*}
so that $\delta^{*}$ dominates $\delta$ and thus $\delta$ is inadmissible.
\item \textbf{Comment on the result of part (b).}

\ \newline
Part (b) shows that if we use a decision rule which violates the SLP but retains the WCP then the corresponding decision rule of the mixture experiment, $\delta$, also violates the SLP as $\delta(1, x_{1}') \neq \delta(2, x_{2}')$. Moreover, this rule is inadmissible and is dominated by a rule, $\delta^{*}$, which does satisfy $\delta^{*}(1, x_{1}') = \delta^{*}(2, x_{2}')$ and so respects the SLP for the outcomes $x_{1}'$, $x_{2}'$. As $\delta$ is inadmissible then we would not want to use it which suggests that violating the SLP is not advisable (if we accept the WCP) or a justification for not applying the WCP is required.
\end{enumerate} 
\end{enumerate}
\section*{Statistical Decision Theory}
\begin{enumerate}\setcounter{enumi}{2}
\item \textbf{Suppose we have a hypothesis test of two simple hypotheses}
\begin{eqnarray*}
H_{0}: X \sim f_{0} & \mbox{versus} & H_{1}: X \sim f_{1}
\end{eqnarray*}
\textbf{so that if $H_{i}$ is true then $X$ has distribution $f_{i}(x)$. It is proposed to choose between $H_{0}$ and $H_{1}$ using the following loss function.}
\begin{eqnarray*}
\begin{array}{cc|cc}
& & \multicolumn{2}{c}{\mbox{Decision}} \\
& & H_{0} & H_{1} \\ \hline 
\mbox{Outcome} & \begin{array}{c} H_{0} \\ H_{1} \end{array} & \begin{array}{c} c_{00} \\ c_{10} \end{array} & \begin{array}{c} c_{01} \\ c_{11} \end{array}
\end{array}
\end{eqnarray*}
\textbf{where $c_{00} < c_{01}$ and $c_{11} < c_{10}$. Thus, $c_{ij} = L(H_{i}, H_{j})$ is the loss when the `true' hypothesis is $H_{i}$ and the decision $H_{j}$ is taken. Show that a decision rule $\delta(x)$ for choosing between $H_{0}$ and $H_{1}$ is admissible if and only if}
\begin{eqnarray*}
\delta(x) & = & \left\{\begin{array}{cl}
H_{0} & \mbox{if } \dfrac{f_{0}(x)}{f_{1}(x)} > c, \\
H_{1} & \mbox{if } \dfrac{f_{0}(x)}{f_{1}(x)} < c, \\
\mbox{either } H_{0} \mbox{ or } H_{1} & \mbox{if } \dfrac{f_{0}(x)}{f_{1}(x)} = c,
\end{array}\right.
\end{eqnarray*}
\textbf{for some critical value $c > 0$.}

\ \newline
For the prior distribution $\pi = (\pi_{0}, \pi_{1})$ where $\pi_{i} > 0$, let $\pi^{*} = (\pi^{*}_{0}, \pi^{*}_{1})$ denote the posterior distribution so that
\begin{eqnarray*}
\pi^{*}_{0} & = & \mathbb{P}(H_{0} \, | \, X = x) \\
& = & \frac{f_{0}(x)\pi_{0}}{f_{0}(x)\pi_{0} + f_{1}(x)\pi_{1}}, \\
\pi^{*}_{1} & = & \mathbb{P}(H_{1} \, | \, X = x) \\
& = & \frac{f_{1}(x)\pi_{1}}{f_{0}(x)\pi_{0} + f_{1}(x)\pi_{1}}.
\end{eqnarray*}
As we also have $f_{i}(x) > 0$ for all $x \in \mathcal{X}$ then $\pi^{*}_{i} > 0$. We calculate the posterior risk under the two decisions $H_{0}$ and $H_{1}$.
\begin{eqnarray}
\rho(\pi^{*}, H_{0}) & = & L(H_{0}, H_{0})\pi^{*}_{0} + L(H_{1}, H_{0})\pi^{*}_{1} \nonumber\\
& = & c_{00}\pi^{*}_{0}+ c_{10}\pi^{*}_{1}, \label{eq:3one} \\
\rho(\pi^{*}, H_{1}) & = & L(H_{0}, H_{1})\pi^{*}_{0} + L(H_{1}, H_{1})\pi^{*}_{1} \nonumber\\
& = & c_{01}\pi^{*}_{0}+ c_{11}\pi^{*}_{1}. \label{eq:3two}
\end{eqnarray}
Thus, 
\begin{eqnarray*}
\rho(\pi^{*}, H_{0}) < \rho(\pi^{*}, H_{1}) & \iff & c_{00}\pi^{*}_{0}+ c_{10}\pi^{*}_{1} < c_{01}\pi^{*}_{0}+ c_{11}\pi^{*}_{1} \\
& \iff & (c_{00}-c_{01})\pi^{*}_{0} < (c_{11}-c_{10})\pi^{*}_{1} \\
& \iff & \frac{\pi^{*}_{0}}{\pi^{*}_{1}} > \frac{c_{11}-c_{10}}{c_{00}-c_{01}} \ = \  \frac{c_{10}-c_{11}}{c_{01}-c_{00}}
\end{eqnarray*}
since $c_{00}-c_{01} < 0$ and $\pi^{*}_{1} > 0$. Using equations (\ref{eq:3one}) and (\ref{eq:3two}) we thus have
\begin{eqnarray}
\rho(\pi^{*}, H_{0}) < \rho(\pi^{*}, H_{1}) & \iff & \frac{f_{0}(x)\pi_{0}}{f_{1}(x)\pi_{1}} >  \frac{c_{10}-c_{11}}{c_{01}-c_{00}} \nonumber \\
& \iff & \frac{f_{0}(x)}{f_{1}(x)} > \frac{(c_{10}-c_{11})\pi_{1}}{(c_{01}-c_{00})\pi_{0}} \ = \ c \label{eq:3three}
\end{eqnarray}
since $\pi_{0}/\pi_{1} > 0$ and thus $c > 0$. The analogous arguments show that 
\begin{eqnarray}
\rho(\pi^{*}, H_{0}) > \rho(\pi^{*}, H_{1}) & \iff & \frac{f_{0}(x)}{f_{1}(x)} < c \label{eq:3four} \\
\rho(\pi^{*}, H_{0}) = \rho(\pi^{*}, H_{1}) & \iff & \frac{f_{0}(x)}{f_{1}(x)} = c \label{eq:3five} 
\end{eqnarray}
The decision rule $\delta(x)$ is chosen to minimise the posterior risk and so is $H_{0}$ when (\ref{eq:3three}) holds, $H_{1}$ when  (\ref{eq:3four}) holds and is indifferent between $H_{0}$ and $H_{1}$ when (\ref{eq:3five}) holds.

\ \newline
Wald's Complete Class Theorem states that a decision rule is admissible if and only if it is a Bayes rule for some prior distribution $\pi$ with strictly positive values. Thus, all admissible decision rules have the form of $\delta(x)$.

\item \textbf{Let $X_{1}, \ldots, X_{n}$ be exchangeable random variables so that, conditional upon a parameter $\theta$, the $X_{i}$ are independent. Suppose that $X_{i} \, | \, \theta \sim N(\theta, \sigma^{2})$ where the variance $\sigma^{2}$ is known, and that $\theta \sim N(\mu_{0}, \sigma_{0}^{2})$ where the mean $\mu_{0}$ and variance $\sigma_{0}^{2}$ are known. We wish to produce a point estimate $d$ for $\theta$, with loss function}
\begin{eqnarray}
L(\theta, d) & = & 1 - \exp\left\{-\frac{1}{2}(\theta - d)^{2} \right\}. \label{eq:loss}
\end{eqnarray}
\begin{enumerate}
\item \textbf{Let $f(\theta)$ denote the probability density function of $\theta \sim N(\mu_{0}, \sigma_{0}^{2})$. Show that $\rho(f, d)$, the risk of $d$ under $f(\theta)$, can be expressed as}
\begin{eqnarray*}
\rho(f, d) & = & 1 - \frac{1}{\sqrt{1+\sigma_{0}^{2}}}\exp\left\{-\frac{1}{2(1+\sigma_{0}^{2})}(d - \mu_{0})^{2} \right\}. 
\end{eqnarray*}
We calculate the risk of decision $d$ under $f(\theta)$,
\begin{eqnarray}
\rho(f, d) & = & \mathbb{E}\left.\left[1 - \exp\left\{-\frac{1}{2}(\theta - d)^{2} \right\} \, \right| \, \theta \sim f(\theta)\right] \nonumber \\
& = & 1 - \mathbb{E}\left.\left[\exp\left\{-\frac{1}{2}(\theta - d)^{2} \right\} \, \right| \, \theta \sim f(\theta)\right] \nonumber \\
& = & 1 - \int_{-\infty}^{\infty} \exp\left\{-\frac{1}{2}(\theta - d)^{2} \right\} \frac{1}{\sqrt{2\pi} \sigma_{0}}\exp\left\{-\frac{1}{2\sigma_{0}^{2}}(\theta - \mu_{0})^{2} \right\} \! d\theta \nonumber \\
& = & 1 - \int_{-\infty}^{\infty} \frac{1}{\sqrt{2\pi} \sigma_{0}}\exp\left\{-\frac{1}{2}\left((\theta - d)^{2} + \frac{1}{\sigma_{0}^{2}}(\theta - \mu_{0})^{2}\right) \right\} \! d\theta. \label{eq:4one}
\end{eqnarray}
Now, using the result that 
\begin{eqnarray*}
(\theta - a)^{2} + b(\theta - c)^{2} & = & (1+b)\left(\theta - \frac{a+bc}{1+b}\right)^{2} + \left(\frac{b}{1+b}\right)(a-c)^{2}
\end{eqnarray*}
for any $a,b,c \in \mathbb{R}$ with $b \neq -1$ we have that
\begin{eqnarray}
(\theta - d)^{2} + \frac{1}{\sigma_{0}^{2}}(\theta - \mu_{0})^{2} & = & \left(\frac{1+\sigma_{0}^{2}}{\sigma_{0}^{2}}\right)\left(\theta - \frac{\sigma_{0}^{2}d+\mu_{0}}{1+\sigma_{0}^{2}}\right)^{2} + \frac{1}{1+\sigma_{0}^{2}}(d-\mu_{0})^{2} \nonumber \\
& = & \left(\frac{1+\sigma_{0}^{2}}{\sigma_{0}^{2}}\right) (\theta - \tilde{\mu})^{2} + \frac{1}{1+\sigma_{0}^{2}}(d-\mu_{0})^{2} \label{eq:4two}
\end{eqnarray}
where $\tilde{\mu} = \dfrac{\sigma_{0}^{2}d+\mu_{0}}{1+\sigma_{0}^{2}}$. Substituting equation (\ref{eq:4two}) into (\ref{eq:4one}) gives
\begin{eqnarray}
\rho(f, d) \ =  \hspace{10cm}  \nonumber \\
 1 - \exp\left\{\frac{-1}{2(1+\sigma_{0}^{2})}(d-\mu_{0})^{2}\right\}\int_{-\infty}^{\infty} \frac{1}{\sqrt{2\pi} \sigma_{0}}\exp\left\{-\frac{1+\sigma_{0}^{2}}{2\sigma_{0}^{2}}(\theta - \tilde{\mu})^{2}\right\} \! d\theta \label{eq:4three}
\end{eqnarray}
We recognise the integrand as a kernel of a $N(\tilde{\mu}, \sigma_{0}^{2}/(1+\sigma_{0}^{2}))$ distribution. Thus, as
\begin{eqnarray*}
\int_{-\infty}^{\infty} \frac{\sqrt{1+\sigma_{0}^{2}}}{\sqrt{2\pi} \sigma_{0}}\exp\left\{-\frac{1+\sigma_{0}^{2}}{2\sigma_{0}^{2}}(\theta - \tilde{\mu})^{2}\right\} \, d\theta & = & 1,
\end{eqnarray*}
equation (\ref{eq:4three}) becomes
\begin{eqnarray*}
\rho(f, d) & = & 1 - \frac{1}{\sqrt{1+\sigma_{0}^{2}}}\exp\left\{-\frac{1}{2(1+\sigma_{0}^{2})}(d - \mu_{0})^{2} \right\}
\end{eqnarray*}
as required.
\item \textbf{Using part (a), show that the Bayes rule of an immediate decision is $d^{*} = \mu_{0}$ and find the corresponding Bayes risk.}

\ \newline
$\rho(f, d)$ is minimised when $\dfrac{1}{\sqrt{1+\sigma_{0}^{2}}}\exp\left\{-\dfrac{1}{2(1+\sigma_{0}^{2})}(d - \mu_{0})^{2} \right\}$ is maximised. This is when $d^{*} = \mu_{0}$. The corresponding Bayes risk is
\begin{eqnarray*}
\rho^{*}(f) \ = \ \rho(f, d^{*}) \ = \ 1 - \frac{1}{\sqrt{1+\sigma_{0}^{2}}}.
\end{eqnarray*}
\item \textbf{Find the Bayes rule and Bayes risk after observing $x = (x_{1}, \ldots, x_{n})$. Express the Bayes rule as a weighted average of $d^{*}$  and the maximum likelihood estimate of $\theta$, $\overline{x} = \frac{1}{n} \sum_{i=1}^{n} x_{i}$, and interpret the weights.}

\ \newline
As $X_{i} \, | \, \theta \sim N(\theta, \sigma^{2})$ then 
\begin{eqnarray*}
f(x \, | \, \theta) & = & \prod_{i=1}^{n} \frac{1}{\sqrt{2\pi} \sigma} \exp\left\{-\frac{1}{2\sigma^{2}} (x_{i} - \theta)^{2}\right\} \\
& \propto & \exp\left\{-\frac{1}{2\sigma^{2}} \sum_{i=1}^{n} (\theta^{2} - 2 x_{i} \theta)\right\} \\
& = & \exp\left\{-\frac{1}{2\sigma^{2}} (n\theta^{2} - 2n \overline{x} \theta)\right\}
\end{eqnarray*}
where the proportionality is with respect to $\theta$. Hence, as $\theta \sim N(\mu_{0}, \sigma_{0}^{2})$, 
\begin{eqnarray*}
f(\theta \, | \, x) & \propto & f(x \, | \, \theta) f(\theta) \\
& \propto & \exp\left\{-\frac{1}{2\sigma^{2}} (n\theta^{2} - 2n \overline{x} \theta)\right\}\exp\left\{-\frac{1}{2\sigma^{2}_{0}}(\theta^{2} - 2 \mu_{0} \theta)\right\} \\
& = & \exp\left\{ - \frac{1}{2}\left(\frac{1}{\sigma_{0}^{2}} + \frac{n}{\sigma^{2}}\right)\left[\theta^{2} - 2\left(\frac{1}{\sigma_{0}^{2}} + \frac{n}{\sigma^{2}}\right)^{-1}\left(\frac{\mu_{0}}{\sigma_{0}^{2}} + \frac{n\overline{x}}{\sigma^{2}}\right)\theta\right]\right\},
\end{eqnarray*}
which we recognise as the kernel of a $N(\mu_{n}, \sigma_{n}^{2})$ where
\begin{eqnarray*}
\mu_{n} \ = \ \left(\frac{1}{\sigma_{0}^{2}} + \frac{n}{\sigma^{2}}\right)^{-1}\left(\frac{\mu_{0}}{\sigma_{0}^{2}} + \frac{n\overline{x}}{\sigma^{2}}\right), & & \sigma_{n}^{2} \ = \ \left(\frac{1}{\sigma_{0}^{2}} + \frac{n}{\sigma^{2}}\right)^{-1}
\end{eqnarray*}
so that $\theta \, | \, x \sim N(\mu_{n}, \sigma_{n}^{2})$. Thus, we have conjugacy. The solution of $[\Theta, \mathcal{D}, f(\theta \, | \, x)$, $L(\theta, d)]$ will be identical to that of $[\Theta, \mathcal{D}, f(\theta), L(\theta, d)]$ but with revised hyperparameters $\mu_{0} \mapsto \mu_{n}$ and $\sigma_{0}^{2} \mapsto \sigma_{n}^{2}$.

\ \newline
The Bayes rule after observing $x$ is thus
\begin{eqnarray*}
d^{*}(x) \ = \ \mu_{n} \ = \ \lambda \mu_{0} + (1- \lambda) \overline{x}
\end{eqnarray*}
where $\lambda = \dfrac{(1/\sigma_{0}^{2})}{(1/\sigma_{0}^{2}) + (n/\sigma^{2})}$. Thus, $d^{*}(x)$ is a weighted average of $d^{*}=\mu_{0}$ and $\overline{x}$ weighted according to their respective precisions. The corresponding Bayes risk is
\begin{eqnarray*}
\rho^{*}(f(\theta \, | \, x)) & = & 1 - \frac{1}{\sqrt{1+\sigma_{n}^{2}}}.
\end{eqnarray*}
\item \textbf{Suppose now, given data $y$, the parameter $\theta$  has the general posterior distribution $f(\theta \, | \, y)$. We wish to use the loss function $L(\theta, d)$, as given in equation (\ref{eq:loss}), to find a point estimate $d$ for $\theta$. By considering an approximation of $L(\theta, d)$, or otherwise, what can you say about the corresponding Bayes rule?}

\ \newline
To first-order, $e^{z} = 1 + z$ so that
\begin{eqnarray*}
L(\theta, d) & \approx & 1 - \left[1- \frac{1}{2}(\theta - d)^{2}\right] \\
& = & \frac{1}{2}(\theta - d)^{2} \\
& \propto & (\theta - d)^{2}.
\end{eqnarray*}
Thus, $L(\theta, d)$ is approximately proportional to quadratic loss and so the Bayes rule may be equivalently found by considering the loss function to be quadratic loss. For the decision problem $[\Theta, \mathcal{D}, \pi(\theta), (\theta-d)^{2}]$ the Bayes rule is $\mathbb{E}(\theta \, | \, \theta \sim \pi(\theta))$ so for $\pi(\theta) = f(\theta \, | \, y)$ the corresponding Bayes rule is $\mathbb{E}(\theta \, | \, Y)$ which is thus the approximate Bayes rule for the loss function given in equation (\ref{eq:loss}).
\end{enumerate}
\end{enumerate}
\section*{Confidence sets and $p$-values}
\begin{enumerate}\setcounter{enumi}{4}
\item \textbf{Show that if $p$ is a family of significance procedures then}
\begin{eqnarray*}
p(x; \Theta_{0}) & = & \sup_{\theta \in \Theta_{0}} p(x; \theta)
\end{eqnarray*}
\textbf{is a significance procedure for the null hypothesis $\Theta_{0} \subset \Theta$, that is that $p(X; \Theta_{0})$ is super-uniform for every $\theta \in \Theta_{0}$.}

\ \newline
Notice that, for all $\theta \in \Theta_{0}$, 
\begin{eqnarray*}
p(x; \Theta_{0}) \leq u & \Longrightarrow & p(x; \theta) \leq u.
\end{eqnarray*}
Thus, by the containment rule, for all $\theta \in \Theta_{0}$,
\begin{eqnarray}
\mathbb{P}(p(X; \Theta_{0}) \leq u \, | \, \theta) & \leq & \mathbb{P}(p(X; \theta) \leq u \, | \, \theta) \label{eq5:one}\\
& \leq & u \label{eq5:two}
\end{eqnarray}
where equation (\ref{eq5:two}) follows from (\ref{eq5:one}) as $p$ is a family of significance procedures. Hence, $p(X; \Theta_{0})$ is super-uniform for every $\theta \in \Theta_{0}$.
\item \textbf{Suppose that, given $\theta$, $X_{1}, \ldots, X_{n}$ are independent and identically distributed $N(\theta, 1)$ random variables so that, given $\theta$, $\overline{X} = \frac{1}{n} \sum_{i=1}^{n} X_{i} \sim N(\theta, 1/n)$.}
\begin{enumerate}
\item \textbf{Consider the test of the hypotheses}
\begin{eqnarray*}
H_{0}: \theta = 0 & \mbox{versus} & H_{1}: \theta = 1
\end{eqnarray*}
\textbf{using the statistic $\overline{X}$ so that large observed values $\overline{x}$ support $H_{1}$. For a given $n$, the corresponding $p$-value is}
\begin{eqnarray*}
p_{n}(\overline{x}; 0) & = & \mathbb{P}(\overline{X} \geq \overline{x} \, | \, \theta = 0).
\end{eqnarray*}
\textbf{We wish to investigate how, for a fixed $p$-value, the likelihood ratio for $H_{0}$ versus $H_{1}$,}
\begin{eqnarray*}
LR(H_{0}, H_{1}) & := & \frac{f(\overline{x} \, | \, \theta = 0)}{f(\overline{x} \, | \, \theta = 1)}
\end{eqnarray*}
\textbf{changes as $n$ increases.}
\begin{enumerate}
\item[(i)] \textbf{Use \R\ to create a plot of $LR(H_{0}, H_{1})$ for each $n \in \{1, \ldots, 20\}$ where, for each $n$, $\overline{x}$ is the value which corresponds to a $p$-value of 0.05.}

\ \newline
For $p = 0.05$, for each $n$, we want to find $\overline{x}$ such that $\mathbb{P}(\overline{X} \geq \overline{x} \, | \, \theta = 0) = 0.05$, that is $\overline{x}$ is the 95th quantile of $N(0, 1/n)$. The following \R\ code can be used to create Figure \ref{figure:one}; a log-scale has been used to present the plot slightly more attractively though this is not necessary.
\begin{verbatim}
alpha <- 0.05
nseq <- 1:20
logBF <- sapply(nseq, function(n){
sd <- 1 / sqrt(n)
z <- qnorm(1 - alpha, mean = 0, sd = sd)
dnorm(z, mean = 0, sd = sd, log = TRUE) -
dnorm(z, mean = 1, sd = sd, log = TRUE)
})
plot(nseq, exp(logBF), type = "b", pch = 16, log = "xy",
ylim = c(0.2, 15),
xlab = "Number of observations, n",
ylab = expression(paste("Likelihood ratio for ", H[0], 
" versus ", H[1])), xpd = NA)
abline(h = 1, lty = 2)
\end{verbatim}
\begin{figure}
\begin{center}
\includegraphics[width=0.75\linewidth]{plot1}
\caption{The likelihood ratio for the hypothesis test $H_{0}: \theta = 0 \ \mbox{versus} \ H_{1}: \theta = 1$ where $\overline{X} \sim N(\theta, 1/n)$ and the $p$-value is fixed at 0.05.\label{figure:one}}
\end{center}
\end{figure}
\item[(ii)] \textbf{Comment on your plot, in particular on what happens to the likelihood ratio as $n$ increases. What is the implication for hypothesis testing and the corresponding (fixed) $p$-value?}

\ \newline
Figure \ref{figure:one} shows that for small $n$ a small $p$-value for $H_{0}$ such as 0.05 corresponds to a likelihood ratio for $H_{0}$ versus $H_{1}$ of less than one, and so `rejecting $H_{0}$ in favour of $H_{1}$ ' is supported by the evidence from the observations. But as $n$ increases a $p$-value of 0.05 for $H_{0}$ comes to correspond to a likelihood ratio that strongly favours $H_{0}$ over $H_{1}$. By the time $n = 20$ the likelihood ratio already exceeds 10.

We conclude that a fixed threshold for a  $p$-value is a very poor way of distinguishing between hypotheses. The moral of this story is that where there is an explicit $H_{1}$ it should be used in a Neyman-Pearson test based on the likelihood ratio and with careful consideration of both size and power. In medical science the `minimal clinically important difference' is the smallest gap between $H_{0}$ and $H_{1}$ that is interesting. It is used to do design calculations for sample size, but it can also be used to do hypothesis testing, rather than just $p$-valuing $H_{0}$.
\end{enumerate}
\item \textbf{Consider the test of the hypotheses}
\begin{eqnarray*}
H_{0}: \theta = 0 & \mbox{versus} & H_{1}: \theta > 0
\end{eqnarray*}
\textbf{using once again $\overline{X}$ as the test statistic.}
\begin{enumerate}
\item[(i)] \textbf{Suppose that $\overline{x} > 0$. Show that}
\begin{eqnarray*}
lr(H_{0}, H_{1}) \ := \ \min_{\theta > 0} \frac{f(\overline{x} \, | \, \theta = 0)}{f(\overline{x} \, | \, \theta)} \ = \ \exp\left\{-\frac{n}{2}\overline{x}^{2}\right\}.
\end{eqnarray*}
Since $\overline{X} \sim N(\theta, 1/n)$ then 
\begin{eqnarray*}
lr(H_{0}, H_{1}) & = & \min_{\theta > 0} \exp\left\{-\frac{n}{2}\left[(\overline{x} - 0)^{2} - (\overline{x} - \theta)^{2}\right]\right\} \\
& = & \exp\left\{-\frac{n}{2}\overline{x}^{2}\right\}
\end{eqnarray*}
if $\overline{x} > 0$ (and is equal to one otherwise). 
\item[(ii)] \textbf{Use \R\ to create a plot of $lr(H_{0}, H_{0})$ for a range of $p$-values for $H_{0}$ from 0.001 to 0.1.}\footnote{The plot doesn't depend upon the actual choice of $n$ and so you may choose $n = 1$.} \textbf{Comment on whether the conventional choice of 0.05 is a suitable threshold for choosing between hypotheses, or whether some other choice might be better.}\footnote{For the origins of the use of 0.05 see Cowles, M. and C. Davis (1982). On the origins of the .05 level of statistical significance. \textit{American Psychologist 37(5)}, 553-558.}  

\ \newline
The aim of this question is for fixed $n$ to investigate how the likelihood ratio changes with the $p$-value. For each $p$-value $\alpha$, $\overline{x}$ is the $100(1-\alpha)$th quantile of $N(0, 1/n)$. The following \R\ code, taking $n=1$, can be used to create Figure \ref{figure:two}.
\begin{verbatim}
pseq <- c(0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1)
z <- qnorm(1 - pseq, mean = 0, sd = 1)
ell <- pmin(1, exp(-(1/2)*z^2))
plot(pseq, ell, type = "b", pch = 16, log = "xy",
xlab = "P-value", ylab = "Lower bound on likelihood ratio")
\end{verbatim}
\begin{figure}
\begin{center}
\includegraphics[width=0.75\linewidth]{plot2}
\caption{Lower bound on the likelihood ratio as a function of the $p$-value for $H_{0}$ for the hypothesis test $H_{0}: \theta = 0 \ \mbox{versus} \ H_{1}: \theta > 0$ where $\overline{X} \sim N(\theta, 1/n)$.\label{figure:two}}
\end{center}
\end{figure}
In this case, a $p$-value of $0.05$ corresponds to a lower bound on the likelihood ratio of $0.26$. If we agree that a likelihood ratio of 1/20 is starting to get interesting, then a $p$-value of a bit less than 0.01 is suggested for this model and these hypotheses.
\end{enumerate}
\end{enumerate}  
\end{enumerate}

\end{document}