
\documentclass[landscape,12pt,titlepage,letterpaper,oneside,final,fleqn]{article}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\usepackage{amsfonts}
\usepackage{amsmath}

\setcounter{MaxMatrixCols}{10}
%TCIDATA{OutputFilter=LATEX.DLL}
%TCIDATA{Version=5.00.0.2570}
%TCIDATA{<META NAME="SaveForMode" CONTENT="3">}
%TCIDATA{Created=Monday, March 15, 2004 08:06:32}
%TCIDATA{LastRevised=Thursday, March 29, 2007 10:19:25}
%TCIDATA{<META NAME="GraphicsSave" CONTENT="32">}
%TCIDATA{<META NAME="DocumentShell" CONTENT="Standard LaTeX\Blank - Standard LaTeX Article">}
%TCIDATA{CSTFile=Overhead Transparencies.cst}
%TCIDATA{PageSetup=72,72,72,72,1}
%TCIDATA{Counters=arabic,1}
%TCIDATA{AllPages=
%H=36
%F=36
%}


\newtheorem{theorem}{Theorem}
\newtheorem{acknowledgement}[theorem]{Acknowledgement}
\newtheorem{algorithm}[theorem]{Algorithm}
\newtheorem{axiom}[theorem]{Axiom}
\newtheorem{case}[theorem]{Case}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{conclusion}[theorem]{Conclusion}
\newtheorem{condition}[theorem]{Condition}
\newtheorem{conjecture}[theorem]{Conjecture}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{criterion}[theorem]{Criterion}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{exercise}[theorem]{Exercise}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{notation}[theorem]{Notation}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{solution}[theorem]{Solution}
\newtheorem{summary}[theorem]{Summary}
\newenvironment{proof}[1][Proof]{\noindent\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\input{tcilatex}

\begin{document}

\title{Multiple Imputation, I}
\author{John M. Abowd}
\date{March, 2007}
\maketitle

\section{Models, Parameters, Bayes Law}

\subsubsection{Symmetric treatment of data and parameters (joint
distribution of the data and the parameters)%
\protect\begin{equation*}
p\left( Y,\protect\theta \right) 
\protect\end{equation*}%
}

\subsubsection{Two factorizations}

Likelihood and prior parameters:

\begin{equation*}
p\left( Y,\theta \right) =p\left( Y\left\vert \theta \right. \right) p\left(
\theta \right)
\end{equation*}

where the first factor, $p\left( Y\left\vert \theta \right. \right) \equiv
\ell \left( \theta \left\vert Y\right. \right) $ is the data model (or
likelihood function) and the second factor $p\left( \theta \right) $ is the
prior distribution of the parameters.

Alternatively, posterior parameters and marginal distribution of the data:%
\begin{equation*}
p\left( Y,\theta \right) =p\left( \theta \left\vert Y\right. \right) p\left(
Y\right)
\end{equation*}

where $p\left( \theta \left\vert Y\right. \right) $ is the posterior
distribution of the parameters (note that $p\left( \theta \left\vert
Y\right. \right) $ is not the same function as $\ell \left( \theta
\left\vert Y\right. \right) $) and $p\left( Y\right) $ is the marginal
(predictive) distribution of the data. By direct maniupulation, we can
derive the posterior distribution of the parameters%
\begin{equation*}
p\left( \theta \left\vert Y\right. \right) =\frac{p\left( Y,\theta \right) }{%
p\left( Y\right) }=\frac{p\left( Y\left\vert \theta \right. \right) p\left(
\theta \right) }{p\left( Y\right) }\equiv \frac{\ell \left( \theta
\left\vert Y\right. \right) p\left( \theta \right) }{p\left( Y\right) }.
\end{equation*}

\pagebreak

\section{Bayes Rule with Missing Data}

Now, consider what happens if we observe some of the data but not other
parts. Partition $Y$ as%
\begin{equation*}
Y=\left( Y_{mis},Y_{obs}\right)
\end{equation*}%
and let the matrix $M$ be defined by%
\begin{equation*}
m_{ij}=\left\{ 
\begin{array}{l}
1,\text{ if }y_{ij}\text{ is missing} \\ 
0,\text{ otherwise}%
\end{array}%
\right.
\end{equation*}%
Then, the joint distribution of interest is%
\begin{equation*}
p\left( Y_{mis},Y_{obs},M,\theta ,\psi \right) =p\left(
Y_{mis},Y_{obs},M\left\vert \theta ,\psi \right. \right) p\left( \theta
,\psi \right)
\end{equation*}%
Decomposing leads to%
\begin{equation*}
p\left( Y_{mis},Y_{obs},M\left\vert \theta ,\psi \right. \right) =p\left(
M\left\vert Y_{mis},Y_{obs},\psi \right. \right) p\left(
Y_{mis},Y_{obs}\left\vert \theta \right. \right)
\end{equation*}%
where the parameters $\psi $ are associated with the missing data mechanism
and the parmeters $\theta $ are associated with the complete data model (or
likelihood function). There can be a functional dependence between $\psi $
and $\theta $, for example $\left( \theta ,\psi \right) =\left( \theta
\left( \beta \right) ,\psi \left( \beta \right) \right) $; so, there is no
loss of generality in this notation. The distribution $p\left( M\left\vert
Y_{mis},Y_{obs},\psi \right. \right) $ is called the missing data generating
mechanism. The distribution $p\left( Y_{mis},Y_{obs}\left\vert \theta
\right. \right) $ is called the complete data model.

\pagebreak

\section{Ignorability (Likelihood based)}

To do inference in the presence of $Y_{mis}$ consider each necessary piece%
\begin{align*}
p\left( Y_{obs},M\left\vert \theta ,\psi \right. \right) & =\dint p\left(
M\left\vert Y_{mis},Y_{obs},\psi \right. \right) p\left(
Y_{mis},Y_{obs}\left\vert \theta \right. \right) dY_{mis} \\
& =p\left( M\left\vert Y_{obs},\psi \right. \right) \dint p\left(
Y_{mis},Y_{obs}\left\vert \theta \right. \right) dY_{mis}
\end{align*}

Likelihood-based ignorability conditions hold:

\begin{itemize}
\item Data missing at random (MAR) $p\left( M\left\vert Y_{mis},Y_{obs},\psi
\right. \right) =p\left( M\left\vert Y_{obs},\psi \right. \right) .$

\item Missing data and model parameters are distinct $\left( \theta ,\psi
\right) \neq \left( \theta \left( \beta \right) ,\psi \left( \beta \right)
\right) $ for some lower dimensional space $\beta .$
\end{itemize}

Bayesian ignorability conditions:

\begin{itemize}
\item Data missing at random (MAR) $p\left( M\left\vert Y_{mis},Y_{obs},\psi
\right. \right) =p\left( M\left\vert Y_{obs},\psi \right. \right) .$

\item The prior distribution of $\left( \theta ,\psi \right) $ factors into $%
p\left( \theta \right) p\left( \psi \right) .$
\end{itemize}

\pagebreak

\section{Imputing Missing Data}

Multiple imputation is based on the posterior predictive distribution of $%
Y_{mis}$. 
\begin{equation*}
p\left( Y_{mis}\left\vert Y_{obs},M,\theta ,\psi \right. \right) =\frac{%
p\left( M\left\vert Y_{mis},Y_{obs},\psi \right. \right) p\left(
Y_{mis},Y_{obs}\left\vert \theta \right. \right) }{p\left(
Y_{obs},M\left\vert \theta ,\psi \right. \right) }
\end{equation*}%
Integrating out the parameters yields%
\begin{equation*}
p\left( Y_{mis}\left\vert Y_{obs},M\right. \right) =\diint \frac{p\left(
M\left\vert Y_{mis},Y_{obs},\psi \right. \right) p\left(
Y_{mis},Y_{obs}\left\vert \theta \right. \right) }{p\left(
Y_{obs},M\left\vert \theta ,\psi \right. \right) }p\left( \theta ,\psi
\left\vert Y_{obs},M\right. \right) d\theta d\psi
\end{equation*}

\pagebreak

\section{Operationalizing the Distribution}

This distribution is the basis for multiple imputation of missing data. To
be useful it must be operationalized considering the same kinds of
assumptions that are discussed above. Imposing MAR and Bayesian ignorability
yields:%
\begin{align*}
p\left( Y_{mis}\left\vert Y_{obs},M\right. \right) & =E_{\left( \theta ,\psi
|Y_{obs},M\right) }\left[ \frac{\text{Joint Distribution}\left(
Y_{mis},Y_{obs},M\right) }{\text{Joint Distribution}\left( Y_{obs},M\right) }%
\right] \\
& =\diint \left[ \frac{p\left( M\left\vert Y_{mis},Y_{obs},\psi \right.
\right) p\left( Y_{mis},Y_{obs}\left\vert \theta \right. \right) }{p\left(
Y_{obs},M\left\vert \theta ,\psi \right. \right) }\right] p\left( \theta
,\psi \left\vert Y_{obs},M\right. \right) d\theta d\psi \\
& =\diint \frac{p\left( M\left\vert Y_{obs},\psi \right. \right) p\left(
Y_{mis},Y_{obs}\left\vert \theta \right. \right) }{p\left( M\left\vert
Y_{obs},\psi \right. \right) \dint p\left( Y_{mis},Y_{obs}\left\vert \theta
\right. \right) dY_{mis}}p\left( \theta \left\vert Y_{obs}\right. \right)
p\left( \psi \left\vert Y_{obs},M\right. \right) d\theta d\psi \\
& =\dint \frac{p\left( Y_{mis},Y_{obs}\left\vert \theta \right. \right) }{%
\dint p\left( Y_{mis},Y_{obs}\left\vert \theta \right. \right) dY_{mis}}%
p\left( \theta \left\vert Y_{obs}\right. \right) d\theta \dint p\left( \psi
\left\vert Y_{obs},M\right. \right) d\psi \\
& =p\left( Y_{mis}\left\vert Y_{obs}\right. \right)
\end{align*}

\pagebreak

\section{Details}

\begin{equation*}
p\left( Y_{obs},M\left\vert \theta ,\psi \right. \right) =p\left(
M\left\vert Y_{obs},\psi \right. \right) \dint p\left(
Y_{mis},Y_{obs}\left\vert \theta \right. \right) dY_{mis}
\end{equation*}%
\begin{equation*}
p\left( Y_{mis}\left\vert Y_{obs}\right. \theta \right) =\frac{p\left(
Y_{mis},Y_{obs}\left\vert \theta \right. \right) }{\dint p\left(
Y_{mis},Y_{obs}\left\vert \theta \right. \right) dY_{mis}}
\end{equation*}%
and%
\begin{equation*}
\dint p\left( \psi \left\vert Y_{obs},M\right. \right) d\psi =1
\end{equation*}%
Note, there are other ways to operationalize the posterior predictive
distribution so that ignorability does not have to be imposed.

\pagebreak

\section{The Bayesian Bootstrap Mulitple Imputation Procedure}

Let $\left( Y_{obs},Y_{mis}\right) $ be partitioned such that observations $%
1,\ldots ,n_{obs}$ correspond to the data in $Y_{obs}$ and $n_{obs}+1,\ldots
,n$ correspond to the data in $Y_{mis}.$

Algorithm:

\begin{itemize}
\item Draw $n_{obs}-1$ random numbers from $U\left( 0,1\right) $. Sorted
from lowest to highest labeled $a_{1},\ldots a_{n_{obs}-1}.$ Define $a_{0}=1$
and $a_{n_{obs}}=1.$

\item Impute each of $n-n_{obs}=n_{mis}$ values of $Y_{mis}$ by sampling
with replacement from $Y_{obs}$ using the probabilities $\left(
a_{1}-a_{0}\right) ,\ldots ,\left( a_{n_{obs}}-a_{n_{obs}-1}\right) $. I.e.,
draw $n_{mis}$ random numbers from $U\left( 0,1\right) $; impute $Y_{i}$
when $a_{i-1}<u\leq a_{i}.$
\end{itemize}

The Bayesian Bootstrap (BB)\ preserves all multivariate relations among the
imputed variables.

The BB can be modified to accomodate non-monotone missing data.

\pagebreak

\section{The Sequential Regression Multivariate Imputation Procedure}

Impose Bayesian ignorability

Estimate $p\left( Y_{mis}|Y_{obs}\right) =\int p\left(
Y_{mis}|Y_{obs},\theta \right) p\left( \theta |Y_{obs}\right) d\theta .$

Sample $M$ times from $p\left( Y_{mis}|Y_{obs}\right) $ as follows%
\begin{equation*}
p\left( y_{1},y_{2},...,y_{K}|\theta _{1},\theta _{2},...,\theta _{K}\right)
=p_{1}\left( y_{1}|\theta _{1}\right) p_{2}\left( y_{2}|y_{1},\theta
_{2}\right) ...p_{K}\left( y_{K}|y_{1},y_{2},...,y_{K-1},\theta _{K}\right)
\end{equation*}%
This equation always holds and is approximated by the SRMI procedure.

The SRMI imputation procedure consists of $L$ rounds. \ Denote the completed
data in round $\ell +1$ on some variable $y_{k}$ by $y_{k}^{(\ell +1)}.$ \
In round $\ell +1,$ missing values of $y_{k}$ are drawn from the predictive
density corresponding to the conditional density:%
\begin{equation}
p_{k}\left( y_{k}|y_{1}^{(\ell +1)},y_{2}^{(\ell +1)},...,y_{k-1}^{(\ell
+1)},y_{k+1}^{(\ell )},...,y_{k}^{(\ell )},\theta _{k}\right)  \label{miss2}
\end{equation}%
where the conditional density $p_{k}$ is specified by an appropriate
generalized linear model, and $\theta _{k}$ are the parameters of that
model. \ Hence under SRMI, at each round $\ell $, the variable under
imputation is regressed on all non-missing data and the most recently
imputed values of missing data. \ The imputation procedure stops after a
predetermined number of rounds or when the imputed values are stable. \
Repeating the procedure $M$ times yields $M$ multiply-imputed data sets.

\pagebreak

\section{General Bayesian Methods for Multiple Imputation}

\subsubsection{Data Augmentation}

These methods use the ignorable missing data hypothesis 
\begin{equation*}
p\left( \theta \left\vert Y_{obs},M\right. \right) =cons\times p\left(
\theta \right) \times \ell \left( \theta \left\vert Y_{obs}\right. \right)
\end{equation*}%
which means that we do not have to estimate $p\left( M\left\vert
Y_{obs},\psi \right. \right) $.

Algorithm for $\ell =0,\ldots $. Let $Y_{mis}^{(0)}=0$ and $\theta ^{(0)}=0$

\begin{itemize}
\item I-step: Draw $Y_{mis}^{(\ell +1)}$ from $p\left( Y_{mis}\left\vert
Y_{obs},\theta ^{(\ell )}\right. \right) $

\item P-step: Draw $\theta ^{(\ell +1)}$ from $p\left( \theta \left\vert
Y_{obs},Y_{mis}^{(\ell )}\right. \right) $
\end{itemize}

As $\ell \rightarrow \infty $, $p^{(\ell )}\left( Y_{mis},\theta \left\vert
Y_{obs}\right. \right) \rightarrow p\left( Y_{mis},\theta \left\vert
Y_{obs}\right. \right) ,$ the joint posterior distribution of the missing
data and the parameters.

\pagebreak

\section{Assessing the Inference Validity of Imputation Procedures}

It is helpful now to specify some variables as complete, $X,$ and other
variables as potentially missing, $Y$. The population consists of $N$
entities with data $\left( X,Y\right) ,$ where $X$ is $N\times q;$ $Y$ is $%
N\times p.$

A variable $Y_{.j}$ is included in the sample (design) as indicated by the
column $I_{.j}.$ So $I$ is $N\times p$ and indicates all of the sampled
entites across all variables. (Recall that complete $X$ means that we have a
well defined frame population will all frame data complete.)

A variable $Y_{.j}$ is observed as indicted by the column $R_{.j}.$ So $R$
is $N\times p$ and indicates all of the responses to all the surveyed
variables.

The term survey can be replaced with record when using an administrative
data base for which a frame population is available.

\pagebreak

\section{Sampling and Missing Data}

$X$ is always observed in all samples.

$Y=\left( Y_{inc},Y_{exc}\right) $ and $R=\left( R_{inc},R_{exc}\right) $
according to $I$. An entity $i$ has been sampled for variable $j$ according
to $I_{ij}=1.$

$Y_{inc}$ is further decomposed into $\left( Y_{obs},Y_{mis}\right) $
according to $R.$ An entity $i$ is observed for variable $j$ according to $%
R_{ij}=1.$

$Y=\left( Y_{obs},Y_{mis},Y_{exc}\right) =\left( Y_{obs},Y_{nob}\right) ,$
where $Y_{nob}=\left( Y_{mis},Y_{exc}\right) $

\pagebreak

\section{Estimands and Their Posterior Distributions}

An estimand is some function of the population data that we wish to
estimate, say $Q\left( X,Y\right) .$

\begin{equation*}
P\left( Q\left\vert X,Y_{obs},R_{inc},I\right. \right) =\int\limits_{\left\{
Y_{nob}\left\vert Q(X,Y)=Q^{\prime }\right. \right\} }P\left(
Y_{nob}\left\vert X,Y_{obs},R_{inc},I\right. \right) dY_{nob}
\end{equation*}

and%
\begin{equation*}
P\left( Y_{nob}\left\vert X,Y_{obs},R_{inc},I\right. \right) =\frac{\int
P\left( X,Y\right) P\left( R\left\vert X,Y\right. \right) P\left(
I\left\vert X,Y,R\right. \right) dR_{exc}}{\iint P\left( X,Y\right) P\left(
R\left\vert X,Y\right. \right) P\left( I\left\vert X,Y,R\right. \right)
dR_{exc}dY_{nob}}
\end{equation*}%
which can, in general, be evaluated from the properties of the sample design
and the assumptions made about the missing data mechanism, e.g., MAR.

\pagebreak

\section{Inference Validity with Missing Data}

\subsection{Population Quantities}

Population value of the estimand and a measure of its variability:%
\begin{equation*}
Q_{0}(X,Y)\text{ and }U_{0}(X,Y)
\end{equation*}

In the complete finite population, these values are given.

\subsubsection{Complete Data Statistics}

\begin{itemize}
\item Estimand: $\hat{Q}(X,Y_{inc},I)$

\item Precision/Variance measure:\ $\hat{U}(X,Y_{inc},I)$
\end{itemize}

\pagebreak

\section{Inference Validity with Missing Data}

\subsection{Multiple Imputation Statistics}

\begin{itemize}
\item Estimand: $\bar{Q}_{L}(X,Y_{obs},I,R_{inc}),$ based on $L$ multiple
imputations $Q_{\ell }\left( X,Y_{\ell }\right) $, the implicate computed
from the $\ell ^{th}$ completed data set%
\begin{equation*}
\bar{Q}_{L}(X,Y_{obs},I,R_{inc})=\frac{1}{L}\sum\limits_{\ell }Q_{\ell
}\left( X,Y_{\ell }\right) 
\end{equation*}

\item Variance measure (within) $\bar{U}_{L}(X,Y_{obs},I,R_{inc}),$ based on 
$L$ multiple imputations using $U\left( X,Y_{\ell }\right) $%
\begin{equation*}
\bar{U}_{L}(X,Y_{obs},I,R_{inc})=\frac{1}{L}\sum\limits_{\ell }U_{\ell
}\left( X,Y_{\ell }\right) 
\end{equation*}

\item Variance measure (between) 
\begin{equation*}
B_{L}(X,Y_{obs},I,R_{inc})=\frac{1}{L-1}\sum\limits_{\ell }\left( Q_{\ell }-%
\bar{Q}_{L}\right) \left( Q_{\ell }-\bar{Q}_{L}\right) ^{\prime }
\end{equation*}

\item Finite population between variance 
\begin{equation*}
B_{\infty }(X,Y_{obs},I,R_{inc})=\lim_{L\rightarrow \infty }\left[ \frac{1}{%
L-1}\sum\limits_{\ell }\left( Q_{\ell }-Q_{0}\right) \left( Q_{\ell
}-Q_{0}\right) ^{\prime }\right] 
\end{equation*}

\item Total variance:%
\begin{equation*}
T_{L}=\bar{U}_{L}(X,Y_{obs},I,R_{inc})+\left( 1+\frac{1}{L}\right)
B_{L}(X,Y_{obs},I,R_{inc})
\end{equation*}

\item Finite population total variance: $T_{0}=U_{0}+B_{\infty }$
\end{itemize}

\pagebreak

\section{Complete Data Must Be Randomization Valid}

If we sample with probability distribution $P\left( I\left\vert X,Y\right.
\right) =P\left( I\left\vert X\right. \right) $,i.e., the sample design
depends only on the data $X$, for which we have a complete frame population,
then the inference system is randomization valid if%
\begin{equation*}
\left( \hat{Q}\left\vert X,Y\right. \right) \sim N(Q_{0}\left( X,Y\right)
,U_{0}\left( X,Y\right) )
\end{equation*}%
and%
\begin{equation*}
\left( \hat{U}\left\vert X,Y\right. \right) \sim (U_{0}\left( X,Y\right)
,\ll U_{0}\left( X,Y\right) )
\end{equation*}%
where $\ll $ means that the matrix tends to $0$ as the finite population
approaches $\infty .$

We assume that the estimand has these properties in complete data so that we
can focus on when the imputation system is randomization valid.

\pagebreak

\section{Randomization Validity of Infinite-$L$ Repeated Multiple Imputation}

A multiple imputation procedure is randomization valid for infinite $L$ if%
\begin{equation*}
\left( \bar{Q}_{\infty }\left\vert X,Y\right. \right) \sim N(Q_{0}\left(
X,Y\right) ,T_{0}\left( X,Y\right) )
\end{equation*}%
and%
\begin{equation*}
\left( T_{\infty }\left\vert X,Y\right. \right) \sim (T_{0}\left( X,Y\right)
,\ll T_{0}\left( X,Y\right) )
\end{equation*}%
where $\bar{Q}_{\infty }$ is $\bar{Q}_{L}(X,Y_{obs},I,R_{inc})$ as $%
L\longrightarrow \infty $, and similarly for $T_{\infty }=U_{\infty
}+B_{\infty }$.

When the conditions for randomization validity hold, the multiple imputation
procedure provides a sampling distribution for $\hat{Q}_{L}$ that is
asymptotically correct as $L\longrightarrow \infty :$%
\begin{equation*}
\left( \hat{Q}_{L}-\bar{Q}_{\infty }\left\vert X,Y_{obs},I,R_{inc}\right.
\right) \sim N\left( 0,T_{\infty }\right)
\end{equation*}%
which allows one to make probability statements about hypothesized values of 
$Q$ in the usual manner.

\end{document}
