236 lines
44 KiB
HTML
236 lines
44 KiB
HTML
<!doctype html>
|
|
<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->
|
|
<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->
|
|
<!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]-->
|
|
<!-- Consider adding a manifest.appcache: h5bp.com/d/Offline -->
|
|
<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->
|
|
<head>
|
|
<meta charset="utf-8">
|
|
|
|
<meta name="perc_linkback" id="perc_linkback" content="FmZlbWNheAdmeAFjZ2x4BmZnYHgTZ2JjbGQ="/> <!-- Use the .htaccess and remove these lines to avoid edge case issues. More info: h5bp.com/i/378 -->
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
|
|
<title>Chapter 12. Description of Ideal Evaluation Methods: Quantitative Approaches to Context Heterogeneity | AHRQ Archive</title>
|
|
<meta name="title" content="Chapter 12. Description of Ideal Evaluation Methods: Quantitative Approaches to Context Heterogeneity | AHRQ Archive">
|
|
<meta name="description" content="This contract final report identifies criteria for assessing the context-sensitive effectiveness and safety of patient safety practices and presents recommendations for future research on this topic.">
|
|
|
|
<!-- Mobile viewport optimized: h5bp.com/viewport -->
|
|
<meta name="viewport" content="width=device-width">
|
|
<!-- link rel="stylesheet" href="/css/style.css" -->
|
|
|
|
<!-- More ideas for your <head> here: h5bp.com/d/head-Tips -->
|
|
<!-- All JavaScript at the bottom, except this Modernizr build.
|
|
Modernizr enables HTML5 elements & feature detects for optimal performance.
|
|
Create your own custom Modernizr build: www.modernizr.com/ -->
|
|
<script src="/js/libs/modernizr-2.5.3.min.js"></script>
|
|
|
|
<!-- "metadata" -->
|
|
<meta name="keywords" content="Agency for Health Care Policy and Research, Agency for Healthcare Research and Quality, AHRQ, AHCPR, access, adverse outcomes, ambulatory care, benchmark, best practices, blood clots, CAHPS, case studies, chronic care, complication, conditions, Consumer Assessment of Health Plans, disparities, error, HAI, health cost, hepatitis, hospital procedure, health care-associated infections, indicators, infection, insurance, measures, medical errors, methodology, nursing, patient care, patient safety, performance measure, quality, study, survey, thrombosis, treatment, urgent care" />
|
|
<meta name="creator" content="Agency for Healthcare Research and Quality (AHRQ)">
|
|
<meta name="datecreated" content="September 2012">
|
|
<meta name="datereviewed" content="November 2014">
|
|
<meta name="language" content="en-us">
|
|
<!-- end="metadata" -->
|
|
<link rel="stylesheet" href="/css/archive.css">
|
|
<link rel="stylesheet" href="/css/ahrqstyleprint_arch.css" media="print">
|
|
<script>
|
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
|
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
|
|
|
|
ga('create', 'UA-75759936-1', 'auto');
|
|
ga(' set', 'anonymizeIp', true);
|
|
|
|
ga('send', 'pageview');
|
|
|
|
</script></head>
|
|
<body><!-- Google Tag Manager -->
|
|
<noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-W4DST4"
|
|
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
|
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
|
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
|
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
|
'//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
|
})(window,document,'script','dataLayer','GTM-W4DST4');</script>
|
|
<!-- End Google Tag Manager -->
|
|
<!-- Prompt IE 6 users to install Chrome Frame. Remove this if you support IE 6.
|
|
chromium.org/developers/how-tos/chrome-frame-getting-started -->
|
|
<!--[if lt IE 7]><p class=chromeframe>Your browser is <em>ancient!</em> <a href="http://browsehappy.com/">Upgrade to a different browser</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to experience this site.</p><![endif]-->
|
|
|
|
<!-- Add your site or application content here -->
|
|
<noscript>Your browser doesn't support JavaScript. Please upgrade to a modern browser or enable JavaScript in your existing browser.</noscript>
|
|
<!-- Page Header v2.0 --><!--Begin Banner CodeS-->
|
|
<div id="ahrqbanner">
|
|
<div class="hhsportion" id="top" name="top">
|
|
<a class="skipnav" href="#h1">Skip Navigation</a>
|
|
<a title="Archive: U.S. Department of Health and Human Services" href="http://archive.hhs.gov/"><img border="0" alt="Archive: U.S. Department of Health and Human Services" src="/images/hhs_banner_arch.gif" width="376" height="49"></a>
|
|
<a class="hhsright" title="U.S. Department of Health and Human Services" href="http://www.hhs.gov/"><img alt="U.S. Department of Health and Human Services" src="/images/hhs_link_arch.gif" width="127" height="41"></a>
|
|
</div>
|
|
<div class="ahrqportion">
|
|
<a title="Archive: Agency for Healthcare Research Quality" href="/"><img border="0" alt="Archive: Agency for Healthcare Research Quality" src="/images/ahrq_banner_arch.jpg" width="566" height="69"></a>
|
|
<form id="banner_searchform" method="get" name="searchForm" action="https://search.ahrq.gov/search">
|
|
<label style="Z-INDEX: -1; POSITION: relative; FONT-SIZE: 0px; MARGIN-RIGHT: -65px" for="search">Search</label>
|
|
<input id="search" class="gotext" onfocus="this.value='';" value=" Search Archive" size="11" type="text" name="q" label="Search archive"/>
|
|
<input type="hidden" name="siteDomain" value="archive.ahrq.gov">
|
|
<input class="gobtn" onclick="javascript:document.searchForm.submit();" alt="Search" src="/images/topbn_GoButton.gif" type="image" name="Submit"/>
|
|
</form><a class="ahrqright" title="AHRQ Home—Live Site" href="http://www.ahrq.gov">www.ahrq.gov</a>
|
|
</div>
|
|
<div class="ahrqlinks">
|
|
<a href="https://www.ahrq.gov/">AHRQ Home—Live Site</a> | <a href="/">Archive Home</a> | <a href="/sitemap.htm">Site Map</a>
|
|
</div>
|
|
</div>
|
|
<div id="PrintBanner"><img alt="Archival print banner" src="/images/printbanner_arch.jpg"/></div><a id="h1" name="h1"></a><!-- End of Page header -->
|
|
<!-- Content Body -->
|
|
<div id="mainContent">
|
|
<!-- Main Content -->
|
|
<!-- Center Content section -->
|
|
<table border="0" cellpadding="0" cellspacing="0" width="100%">
|
|
<tbody><tr valign="top">
|
|
<td width="70%">
|
|
<!-- Center Content section -->
|
|
<table style="margin-left: 10px; margin-right: 10px; margin-top: 5px;" summary="This table gives the layout format of the bread crumb area and the center content area." border="0" cellpadding="0" cellspacing="0" width="100%">
|
|
<!--DWLayoutTable-->
|
|
<tbody><tr>
|
|
<td class="crumb_link"><div id="crumbContent">
|
|
</span></p>
|
|
</div>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td>
|
|
<div id="centerContent">
|
|
<div class="headnote">
|
|
<p>This information is for reference purposes only. It was current when produced and may now be outdated. Archive material is no longer maintained, and some links may not work. Persons with disabilities having difficulty accessing this information should contact us at: <a href="https://info.ahrq.gov/">https://info.ahrq.gov</a>. Let us know the nature of the problem, the Web address of what you want, and your contact information.</p>
|
|
<p>Please go to <a href="https://www.ahrq.gov/">www.ahrq.gov</a> for current information.</p>
|
|
</div>
|
|
<!-- T:ahrqPgPublication -->
|
|
<div role="main" id="page-container">
|
|
<div id="breadcrumb-container">
|
|
<div id="breadcrumbs" class="container_12">
|
|
<!-- T:SnBreadcrumbs -->
|
|
|
|
|
|
|
|
</div>
|
|
</div>
|
|
<div id="toolbar-container">
|
|
<div id="pub-toolbar" class="container_12">
|
|
<!-- T:ahrqSnPubNav -->
|
|
<div id="prev-next-buttons">
|
|
<a href="/research/findings/final-reports/contextsensitive/context11.html" class="prev-button"><span>Previous Page</span></a>
|
|
<a href="/research/findings/final-reports/contextsensitive/context13.html" class="next-button"><span>Next Page</span></a>
|
|
</div>
|
|
<div id="pub-utility-buttons">
|
|
<ul id="utility-groups">
|
|
<li class="utility content-button">
|
|
<a href="#" class=""><span>Table of Contents</span></a>
|
|
<div class="toc-dropdown">
|
|
<div class="arrow-up"></div>
|
|
<div class="toc-content">
|
|
</div>
|
|
</div>
|
|
</li>
|
|
<li class="utility download-button"><a href="#"><span>Download</span></a><div class="dl-dropdown"><div class="arrow-up"></div><div class="dl-content"><ul><li>
|
|
<span><a href="/research/findings/final-reports/contextsensitive/context.pdf">Assessing the Evidence for Context-Sensitive Effectiveness and Safety </a> [ <img src="/resources/file_logos/pdf.gif" alt="PDF file" title="PDF file" class="icon" /> - 711.65 KB]</span>
|
|
</li></ul></div></div></li> </ul>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div id="page-backdrop" class="container_12">
|
|
<div class="page">
|
|
<div id="left-col" class="alpha grid_3">
|
|
<nav id="left"><!-- T:SnleftNav -->
|
|
</nav> </div>
|
|
<!--<div id="content-col" class="grid_9">-->
|
|
<article class="grid_9">
|
|
<!-- Right Rail -->
|
|
<aside class="three-col">
|
|
<!-- T:ahrqSnPubCover -->
|
|
<div id="pub-nums">
|
|
<ul>
|
|
<li>Publication # 11-0006-EF</li> </ul>
|
|
</div>
|
|
|
|
</aside>
|
|
<!-- Center Rail -->
|
|
<h1>Chapter 12. Description of Ideal Evaluation Methods: Quantitative Approaches to Context Heterogeneity</h1>
|
|
<h2>Assessing the Evidence for Context-Sensitive Effectiveness and Safety </h2> <div id="basic-modal"><!-- start: Basic Modal -->
|
|
<p><strong>Special contribution from Naihua Duan, Ph.D., Columbia University, New York, NY</strong></p><h3>Introduction</h3><p>Context often moderates intervention effectiveness; i.e., the effectiveness of an intervention might vary from site to site, depending on the contextual factors present at each site.<sup><a href="#ref1">1,2</a></sup> This phenomenon is what we have termed context heterogeneity. This moderation effect is usually formulated statistically through the "intervention  context" interaction:</p><p>(1a) Yi = b<sub>0</sub> + b<sub>1</sub>  Ti + b<sub>2</sub>  Ci + b<sub>12</sub>  Ti  Ci + <em>epsilon</em>i,</p><p>where i denotes the unit of analysis (usually the various sites in the study, but it can also be dyads of sites in matched comparisons), Yi denotes the outcome measure, Ti denotes the intervention status (Ti=1 for intervention, Ti=0 for control), Ci denotes the contextual factor, Ti  Ci denotes the "intervention  context" interaction, <em>epsilon</em>i denotes random error, b<sub>0</sub> denotes the intercept for the model, b<sub>1</sub> denotes the main effect for the intervention, b<sub>2</sub> denotes the main effect for the contextual factor, and b<sub>12</sub> denotes the moderation effect for the contextual factor, i.e., the influence of the contextual factor on intervention effectiveness.</p><p>As an example, consider a dichotomous contextual factor, say, C=1 denotes a teaching hospital and C=0 denotes a non-teaching hospital. According to model (1a), the intervention effect for a non-teaching hospital is given by b<sub>1</sub>, while the intervention effect for a teaching hospital is given by b<sub>1</sub> + b<sub>12</sub>. If the moderation effect is absent (b<sub>12</sub>=0), the intervention effect does not vary between teaching and non-teaching hospitals. If the moderation effect is present (b<sub>12</sub>≠0), the intervention effect does vary between teaching and non-teaching hospitals, the difference being the moderation effect b<sub>12</sub>.</p><p>Model (1a) presents the "intervention  context" interaction" for a single contextual factor. The model can be generalized in a straightforward manner to accommodate multiple contextual factors:</p><p>(1b) Yi = b<sub>0</sub> + b<sub>1</sub>  Ti + b<sub>2</sub>  C<sub>2i</sub> + b<sub>3</sub> x C<sub>3i</sub> + ... + b<sub>k</sub> x C<sub>ki</sub><br />   + b<sub>12</sub>  Ti  C<sub>2i</sub> + b<sub>13</sub>  Ti  C<sub>3i</sub> + ... + b<sub>1k</sub>  Ti  C<sub>ki</sub> + <em>epsilon</em>i,</p><p>where C<sub>2i</sub>, C<sub>3i</sub>,..., C<sub>ki</sub> denote multiple contextual factors, and b<sub>12</sub>, b<sub>13</sub>,..., b<sub>1k</sub> denote the respective moderation effects.</p><p>The assessment of the moderation effect depends on the methodology used to assess the intervention effect. We discuss here eight scenarios.</p><h3>Pre-Post Comparisons</h3><p>One option that could be used to assess the intervention effect is to compare the outcome measures pre- and post-intervention, without concurrent control sites. Under the assumption that the outcome measure is stable over time if no intervention were provided, any change observed would be attributed to the effect of the intervention. In particular, Model (1a) takes the following form:</p><p>(2a) Y<sub>0i</sub> = b<sub>0</sub> + b<sub>2</sub>  C<sub>i</sub> + <em>epsilon</em><sub>0i</sub>,</p><p>(2b) Y<sub>1i</sub> = b<sub>0</sub> + b<sub>1</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>12</sub>  C<sub>i</sub> + <em>epsilon</em><sub>1i</sub>,</p><p>(2c) Y<sub>1i</sub> - Y<sub>0i</sub> = b<sub>1</sub> + b<sub>12</sub>  C<sub>i</sub> + (<em>epsilon</em><sub>1i</sub> - <em>epsilon</em><sub>0i</sub>),</p><p>where the subscript <sub>i</sub> denotes the i-th site in the study, Y<sub>0i</sub> denotes the pre-intervention outcome measure at the i-th site, Y<sub>1i</sub> denotes the post-intervention outcome measure at the i-th site; C<sub>i</sub> denotes the contextual factor at the i-th site; <em>epsilon</em><sub>0i</sub> and <em>epsilon</em><sub>1i</sub> denote the respective error terms for the pre- and post-intervention outcome measures. Compared to Model (1a), the terms b<sub>1</sub>  T and b<sub>12</sub>  T  C are absent in submodel (2a) because the intervention status T assumes the value T=0 under the control condition. Similarly, in submodel (2b) compared to Model (1a), the intervention status T assumes the value T=1 under the intervention condition, therefore the terms b<sub>1</sub>  T and b<sub>12</sub>  T  C are given as b<sub>1</sub> and b<sub>12</sub>  C<sub>i</sub>. Submodel (2c) compares submodels (2a) and (2b): the term Y<sub>1i</sub> - Y<sub>0i</sub> denotes the pre-post change, which measures the intervention effect at the i-th site.</p><p>The moderation effect (b<sub>12</sub> in Model (2c)) can be assessed by regressing the intervention effect at the i-th site, Y<sub>1i</sub> - Y<sub>0i</sub>, on the contextual factor C<sub>i</sub> in model (2c). For continuous contextual factors, this regression analysis estimates the rate of change for the intervention effect at the i-th site, Y<sub>1i</sub> - Y<sub>0i</sub>, when the contextual factor C<sub>i</sub> changes by one unit. For dichotomous contextual factors, this regression analysis simplified to a two-sample comparison, comparing the average of the intervention effect among sites with the contextual factor C<sub>i</sub>=1 (such as teaching hospitals) versus the average of the intervention effect among sites with the contextual factor C<sub>i</sub>=0 (such as non-teaching hospitals). </p><p>The validity of pre-post comparisons depends on the validity of the assumption that the outcome measure is stable over time. If this assumption is questionable, e.g., if there is a possibility of a secular trend in the outcome measures, the validity of pre-post comparisons is questionable both for the assessment of intervention effect per se, and for the assessment of the effect of moderation for the contextual factors.</p><h3>Longitudinal Comparisons</h3><p>An important extension of pre-post comparisons is longitudinal comparisons of repeated measurements of the outcome measures over time, without concurrent control sites. Under the assumption that the outcome measure is stable over time if no intervention were provided, any change over time that is observed would be attributed to the effect of the intervention. In particular, Model (1a) takes the following form:</p><p>(3a) Y<sub>0i</sub> = b<sub>0</sub> + b<sub>2</sub>  C<sub>i</sub> + <em>epsilon</em><sub>0i</sub>,</p><p>(3b) Y<sub>ti</sub> = b<sub>0</sub> + b<sub>1</sub>  t + b<sub>2</sub>  C<sub>i</sub> + b<sub>12</sub>  C<sub>i</sub>  t + <em>epsilon</em><sub>ti</sub>,</p><p>(3c) R<sub>i</sub> = b<sub>1</sub> + b<sub>12</sub>  C<sub>i</sub> + <em>delta</em><sub>i</sub>,</p><p>where the subscript <sub>i</sub> denotes the i-th site in the study, Y<sub>0i</sub> denotes the pre-intervention outcome measure at the i-th site, Y<sub>ti</sub> denotes the outcome measure at time t for the i-th site; R<sub>i</sub> denotes the rate of change for the outcome measure for the i-th site; C<sub>i</sub> denotes the contextual factor at the i-th site; <em>epsilon</em><sub>0i</sub> and <em>epsilon</em><sub>ti</sub> denote the respective error terms for the outcome measures; <em>delta</em><sub>i</sub> denotes the error term for the rate of change. We assume here that the trajectory of the outcome measure is linear over time, therefore the influence of time on the outcome measure in Model (3b) can be expressed as linear functions in time, t. Furthermore, the linearity assumption allows us to summarize the trajectory using the rate of change, R, in Model (3c): the rate of change for the i-th site, R<sub>i</sub>, can be estimated by regressing the outcome measures, Y<sub>ti</sub>'s, on time, t, within the i-th site. It is of course possible to extend the model beyond linear trajectories and allow non-linear trajectories.</p><p>In Model (3c), b<sub>1</sub> measures the intervention effect for sites with null values for the contextual factor, such as non-teaching hospitals; for these sites, the outcome measures improve at the rate of b<sub>1</sub> per unit time. For sites with C=1, say, teaching hospitals, the intervention effect is b<sub>1</sub> + b<sub>12</sub> - the term b<sub>12</sub> measures the moderation effect for the contextual factor C. </p><p>The moderation effect (b<sub>12</sub> in Model (3c)) can be assessed by regressing the rate of change, R<sub>i</sub>, on the contextual factor C<sub>i</sub> in model (3c).</p><p>The validity of longitudinal comparisons depends on the validity of the assumption that the outcome measure is stable over time. If this assumption is questionable, e.g., if there is a possibility of a secular trend in the outcome measures, the validity of longitudinal comparisons is questionable both for the assessment of intervention effect per se, and for the assessment of the effect of moderation for the contextual factors.</p><h3>Matched Comparisons for Post-intervention Outcome Measures</h3><p>Another strategy that could be used to assess the intervention effect is to include concurrent control sites matched individually to the intervention sites and compare the post-intervention outcome measures across sites.<sup><a href="#notea">a</a></sup> We assume that the sites are matched on the contextual factor. Under the assumption that the matched sites differ only in the intervention status, the intervention effect can be assessed by the difference in the post-intervention outcome measures for each dyad of matched sites. In particular, Model (1a) takes the following form under this approach:</p><p>(4a) Y<sub>0i</sub> = b<sub>0</sub> + b<sub>2</sub>  C<sub>i</sub> + <em>epsilon</em><sub>0i</sub>,</p><p>(4b) Y<sub>1i</sub> = b<sub>0</sub> + b<sub>1</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>12</sub>  C<sub>i</sub> + <em>epsilon</em><sub>1i</sub>,</p><p>(4c) Y<sub>1i</sub> - Y<sub>0i</sub> = b<sub>1</sub> + b<sub>12</sub>  C<sub>i</sub> + (<em>epsilon</em><sub>1i</sub> - <em>epsilon</em><sub>0i</sub>),</p><p>where the subscript <sub>i</sub> denotes the i-th dyad of matched sites in the study, Y<sub>0i</sub> denotes the post-intervention outcome measure at the control site in the i-th dyad, Y<sub>1i</sub> denotes the post-intervention outcome measure at the intervention site in the i-th dyad; C<sub>i</sub> denotes the contextual factor for both sites in the i-th dyads; <em>epsilon</em><sub>0i</sub> and <em>epsilon</em><sub>1i</sub> denote the respective error terms for the post-intervention outcome measures. Compared to Model (1a), the terms b<sub>1</sub>  T and b<sub>12</sub>  T  C are absent in submodel (4a) because the intervention status T assumes the value T=0 for the control site. Similarly, in submodel (4b) compared to Model (1a), the intervention status T assumes the value T=1 for the intervention site, therefore the terms b<sub>1</sub>  T and b<sub>12</sub>  T  C are given as b<sub>1</sub> and b<sub>12</sub>  C<sub>i</sub>. Submodel (4c) compares submodels (4a) and (4b): the term Y<sub>1i</sub> - Y<sub>0i</sub> denotes the difference between the intervention and control sites in the i-th dyad, which measures the intervention effect in the i-th dyad.</p><p>The moderation effect (b<sub>12</sub> in Model (4c)) can be assessed by regressing the intervention effect in the i-th dyad, Y<sub>1i</sub> - Y<sub>0i</sub>, on the contextual factor C<sub>i</sub> in model (4c). For continuous contextual factors, this regression analysis estimates the rate of change for the intervention effect in the i-th dyad, Y<sub>1i</sub> - Y<sub>0i</sub>, when the contextual factor C<sub>i</sub> changes by one unit. For dichotomous contextual factors, this regression analysis simplified to a two sample comparison, comparing the average of the intervention effect among dyads with the contextual factor C<sub>i</sub>=1 (such as teaching hospitals) versus the average of the intervention effect among dyads with the contextual factor C<sub>i</sub>=0 (such as non-teaching hospitals). </p><p>The validity of matched comparisons depends on the validity of the assumption that the matched sites differ only in the intervention status. If this assumption is questionable, e.g., if there are important prognostic factors that differ between matched sites in the same dyad, the validity of matched comparisons is questionable, both for the assessment of intervention effect per se, and for the assessment of the effect of moderation for the contextual factors. If the unmatched prognostic factors are observed, it is possible to adjust for them using analysis of covariance (ANCOVA) models for post-intervention outcomes, or propensity scores analyses, to be discussed in Section F below.</p><h3>Matched Comparisons for Pre-Post-Intervention Changes in Outcome Measures</h3><p>Another strategy that can be used to assess the intervention effect is to combine strategies (A) and (B) and obtain both pre- and post-intervention outcomes measures for both intervention and control sites. The pre-post changes in outcome measures are compared across intervention and control sites to assess intervention effects. By combining pre-post and matched site comparisons, this approach can be applied under weaker assumptions than the assumptions required for either strategy discussed in Sections A and B. In particular, this combined approach no longer requires the rather strong assumption in Section A that there is no secular trend. Instead, this approach only requires that any secular trend that might be present be the same between intervention and control sites in the same dyad. This assumption is also weaker than the rather strong assumption in Section B that the matched sites in the same dyad differ only in the intervention status. Instead, this approach allows the sites to differ in their pre-intervention status as long as these differences do not affect the pre-post change. In particular, Model (1a) takes the following form under this approach:</p><p>(5a) D<sub>0i</sub> = b<sub>0</sub> + b<sub>2</sub>  C<sub>i</sub> + <em>epsilon</em><sub>0i</sub>,</p><p>(5b) D<sub>1i</sub> = b<sub>0</sub> + b<sub>1</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>12</sub>  C<sub>i</sub> + <em>epsilon</em><sub>1i</sub>,</p><p>(5c) D<sub>1i</sub> - D<sub>0i</sub> = b<sub>1</sub> + b<sub>12</sub>  C<sub>i</sub> + (<em>epsilon</em><sub>1i</sub> - <em>epsilon</em><sub>0i</sub>),</p><p>where the subscript <sub>i</sub> denotes the i-th dyad of matched sites in the study, C<sub>i</sub> denotes the contextual factor for both sites in the i-th dyads. In submodel (5a), D<sub>0i</sub> denotes the pre-post change in the outcome measure at the control site in the i-th dyad, which measures the secular trend in the i-th dyad. Here we allow the secular trend to depend on the contextual factor. In submodel (5b), D<sub>1i</sub> denotes the pre-post change in the outcome measure at the intervention site in the i-th dyad. Submodel (5c) compares submodels (5a) and (5b): the term D<sub>1i</sub> - D<sub>0i</sub> denotes the difference in the pre-post change between the intervention and control sites in the i-th dyad, which measures the intervention effect in the i-th dyad.</p><p>The moderation effect (b<sub>12</sub> in Model (5c)) can be assessed by regressing the intervention effect in the i-th dyad, D<sub>1i</sub> - D<sub>0i</sub>, on the contextual factor C<sub>i</sub> in model (5c). For continuous contextual factors, this regression analysis estimates the rate of change for the intervention effect in the i-th dyad, D<sub>1i</sub> - D<sub>0i</sub>, when the contextual factor C<sub>i</sub> changes by one unit. For dichotomous contextual factors, this regression analysis simplified to a two-sample comparison, comparing the average of the intervention effect among dyads with the contextual factor C<sub>i</sub>=1 (such as teaching hospitals) versus the average of the intervention effect among dyads with the contextual factor C<sub>i</sub>=0 (such as non-teaching hospitals). </p><p>The validity of matched comparisons depends on the validity of the assumption that any secular trend that might be present be the same between intervention and control sites in the same dyad. If this assumption is questionable, the validity of matched comparisons is questionable both for the assessment of intervention effect per se, and for the assessment of the effect of moderation for the contextual factors. If the unmatched prognostic factors associated with the secular trend are observed, it is possible to adjust for them using analysis of covariance (ANCOVA) models for pre-post changes, or propensity scores analyses, to be discussed below.</p><h3>Matched Comparisons for Longitudinal Rates of Change in Outcome Measures</h3><p>A combined strategy similar to matched comparison of pre-post change, discussed in Section D above, is to combine strategies (A) and (C) and assess longitudinally the rate of change for outcomes measures for both intervention and control sites. The rate of change is compared across intervention and control sites to assess intervention effects. By combining longitudinal and matched sites comparisons, this approach can be applied under weaker assumptions than the assumptions required for either option discussed in Sections A and C. In particular, Model (1a) takes the following form under this approach:</p><p>(6a) R<sub>0i</sub> = b<sub>0</sub> + b<sub>2</sub>  C<sub>i</sub> + <em>delta</em><sub>0i</sub>,</p><p>(6b) R<sub>1i</sub> = b<sub>0</sub> + b<sub>1</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>12</sub>  C<sub>i</sub> + <em>delta</em><sub>1i</sub>,</p><p>(6c) R<sub>1i</sub> - R<sub>0i</sub> = b<sub>1</sub> + b<sub>12</sub>  C<sub>i</sub> + (<em>delta</em><sub>1i</sub> - <em>delta</em><sub>0i</sub>),</p><p>where the subscript <sub>i</sub> denotes the i-th dyad of matched sites in the study, C<sub>i</sub> denotes the contextual factor for both sites in the i-th dyads. In submodel (6a), R<sub>0i</sub> denotes the longitudinal rate of change in the outcome measure at the control site in the i-th dyad, which measures the secular trend in the i-th dyad. Here we allow the secular trend to depend on the contextual factor. In submodel (5b), R<sub>1i</sub> denotes the longitudinal change in the outcome measure at the intervention site in the i-th dyad. Submodel (6c) compares submodels (6a) and (6b): the term R<sub>1i</sub> - R<sub>0i</sub> denotes the difference in the longitudinal rate of change between the intervention and control sites in the i-th dyad, which measures the intervention effect in the i-th dyad.</p><p>The moderation effect (b<sub>12</sub> in Model (6c)) can be assessed by regressing the intervention effect in the i-th dyad, R<sub>1i</sub> - R<sub>0i</sub>, on the contextual factor C<sub>i</sub> in model (6c).</p><p>The validity of matched comparisons depends on the validity of the assumption that any secular trend that might be present be the same between intervention and control sites in the same dyad. If this assumption is questionable, the validity of matched comparisons is questionable both for the assessment of intervention effect per se, and for the assessment of the effect of moderation for the contextual factors. If the unmatched prognostic factors associated with secular trend are observed, it is possible to adjust for them using analysis of covariance (ANCOVA) models for longitudinal rate of change, or propensity scores analyses, discussed below.</p><h3>Adjusted Comparisons for Post-intervention Outcome Measures</h3><p>Matched comparisons discussed in Sections (B)-(E) above assume that the intervention and control sites can be matched to the degree required under each strategy. In practical applications, this usually is not a realistic assumption. Therefore, adjustment for covariates is usually important, both for studies in which matching is attempted and for studies in which matching is not attempted. The adjustments can made either using the analysis of covariance (ANCOVA) model, or the propensity scores analysis.<sup><a href="#ref3">3-6</a></sup></p><p>With ANCOVA, Model (1a) takes the following form:</p><p>(7a) Y<sub>i</sub> = b<sub>0</sub> + b<sub>1</sub>  T<sub>i</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>3</sub>  W<sub>i</sub> + b<sub>12</sub>  T<sub>i</sub>  C<sub>i</sub> + <em>epsilon</em><sub>i</sub>,</p><p>where the subscript <sub>i</sub> denotes the i-th site in the study, Y<sub>i</sub> denotes the post-intervention outcome measure for the i-th site, T<sub>i</sub> denotes the intervention condition for the i-th site (T=1 if intervention, T=0 if control), C<sub>i</sub> denotes the contextual factor for the i-th site, W<sub>i</sub> denotes the covariates for the i-th site, and <em>epsilon</em><sub>i</sub> denotes the error term. The inclusion of the term b<sub>3</sub>  W<sub>i</sub> adjusts for the imbalance in the covariates, W, that might be present between the intervention vs. control sites.</p><p>The coefficient b<sub>1</sub> denotes the intervention effect for sites with null values of the contextual factor (C=0), such as non-teaching hospitals; the intervention effect for sites with the value off the contextual factor C=1, such as teaching hospitals, is given by b<sub>1</sub> + b<sub>12</sub>. The coefficient b12 denotes the moderation effect, e.g., how the intervention effect differs between teaching hospitals and non-teaching hospitals.</p><p>The moderation effect (b<sub>12</sub> in Model (7)) can be assessed (along with the other coefficients in the model) by regressing the post-intervention outcome measure, Y<sub>i</sub>, on the intervention status T<sub>i</sub>, the contextual factor C<sub>i</sub>, the covariates, W<sub>i</sub>, and the interaction term, T<sub>i</sub>  C<sub>i</sub>, in model (7).</p><p>With propensity score analysis, we first model the propensity for the i-th site to be an intervention site:</p><p>(7b) <em>Pi</em> <sub>i</sub> = logit(P(T<sub>i</sub>=1)) = g0 + g1  C<sub>i</sub> + g2  W<sub>i</sub>.</p><p>The propensity model (7) is usually specified and fitted as a logistic regression of intervention status (T) on the contextual factor (C) and covariates (W). The fitted model is then applied to all sites in the study to derive the propensity score, <em>Pi</em> , for each site to be an intervention site. The propensity scores can then be used in several ways to adjust for the imbalance between the intervention and control sites in the sample. One option that is particularly suitable for the assessment of the moderation effect for the context factor, C, is the following ANCOVA model that uses the propensity score <em>Pi</em> instead of the covariates W in model (7a):</p><p>(7c) Y<sub>i</sub> = b<sub>0</sub> + b<sub>1</sub>  T<sub>i</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>3</sub>  <em>Pi</em> <sub>i</sub> + b<sub>12</sub>  T<sub>i</sub>  C<sub>i</sub> + <em>epsilon</em><sub>i</sub>.</p><p>Alternative ways that can be used to implement the propensity score analysis include matching, stratification, and weighting.</p><p>The validity of adjusted comparisons depends on the success of the adjustment to remove all imbalances between intervention and control sites in the sample. With either ANCOVA or propensity scores analysis, it is necessary to assume that all relevant covariates are observed, i.e., there are no hidden confounders.</p><h3>Adjusted Comparisons for Pre-post-intervention Changes in Outcome Measures</h3><p>Another strategy is to combine strategies (A) and (F), and apply ANCOVA or propensity score analysis to the pre-post changes in outcome measures.</p><p>With ANCOVA, Model (1a) takes the following form:</p><p>(8a) D<sub>i</sub> = b<sub>0</sub> + b<sub>1</sub>  T<sub>i</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>3</sub>  W<sub>i</sub> + b<sub>12</sub>  T<sub>i</sub>  C<sub>i</sub> + <em>epsilon</em><sub>i</sub>,</p><p>where D<sub>i</sub> denotes the pre-post change for the i-th site. The rest of the model is identical to Model (7a) discussed in Section F above.</p><p>With propensity score analysis, the same propensity model (7b) is used to assess the propensity scores <em>Pi</em> <sub>i</sub>. The fitted propensity scores are then used in the following model:</p><p>(8c) D<sub>i</sub> = b<sub>0</sub> + b<sub>1</sub>  T<sub>i</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>3</sub>  <em>Pi</em> <sub>i</sub> + b<sub>12</sub>  T<sub>i</sub>  C<sub>i</sub> + <em>epsilon</em><sub>i</sub>.</p><h3>Adjusted Comparisons for Longitudinal Rates of Change in Outcome Measures</h3><p>A combined strategy similar to adjusted comparison of pre-post change, discussed above, is to combine strategies (A) and (E) and compare longitudinally the rate of change for outcomes measures between intervention and control sites, adjusted for covariates that might be imbalanced, using either ANCOVA or propensity scores analysis.</p><p>With ANCOVA, model (1a) takes the following form:</p><p>(9a) R<sub>i</sub> = b<sub>0</sub> + b<sub>1</sub>  T<sub>i</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>3</sub>  W<sub>i</sub> + b<sub>12</sub>  T<sub>i</sub>  C<sub>i</sub> + <em>epsilon</em><sub>i</sub>,</p><p>where R<sub>i</sub> denotes the pre-post change for the i-th site. The rest of the model is identical to Model (7a) discussed above.</p><p>With propensity score analysis, the same propensity model (7b) is used to assess the propensity scores <em>Pi</em> <sub>i</sub>. The fitted propensity scores are then used in the following model:</p><p>(9c) R<sub>i</sub> = b<sub>0</sub> + b<sub>1</sub>  T<sub>i</sub> + b<sub>2</sub>  C<sub>i</sub> + b<sub>3</sub>  <em>Pi</em> <sub>i</sub> + b<sub>12</sub>  T<sub>i</sub>  C<sub>i</sub> + <em>epsilon</em><sub>i</sub>.</p><p>The choice of analytic strategies for the assessment of intervention effect and the corresponding strategies for the assessment of the moderation effect for contextual factors depend on the design of the study. Strategies that are based on pre-post changes or longitudinal rates of changes can be applied only to studies that obtain pre-post measures or repeated measures of outcomes. Strategies that are based on matched comparisons can be applied only to studies designed with matched sites. In order to allow more flexibility in the analytic strategies, it would be advantageous to design the studies to include these features (either pre-post measures of outcome, or, more preferably, repeated measures; and matched sites).</p><p>The strategies discussed above are not exhaustive. Some of the strategies can be expanded, e.g., the linearity assumption in the longitudinal models can be relaxed to allow for non-linear trajectories over time. In addition, strategies such as instrumental variables analysis<sup><a href="#ref7">7</a></sup> and causal sensitivity analysis<sup><a href="#ref6">6</a>,<a href="#ref8">8</a></sup> can be used to address hidden bias, i.e., unobserved factors that are imbalanced between intervention and control sites. However, the eight strategies discussed above are probably the most practical methods and most commonly applied.</p><h3>References for Chapter 12</h3><ol><li><a id="ref1" name="ref1"></a> Kravitz R, Duan N, Braslow JT. Evidence based medicine: Heterogeneity of treatment effects, and the trouble with averages. <em>Milbank Q</em> 2004; 82(4):66187. Erratum in: <em>Milbank Q</em> 2006; 84(4):759-60.</li><li><a id="ref2" name="ref2"></a> Greenfield S, Kravitz R, Duan N, Kaplan SH. Heterogeneity of treatment effects: Implications for guidelines, payment, and quality assessment. <em>Am J Med</em> 2007; 120(4 Suppl 1):S3-9.</li><li><a id="ref3" name="ref3"></a> Rosenbaum PR, Rubin DB. The central role of propensity score in observational studies for causal effects. <em>Biometrika</em> 1983; 70:41-55.</li><li><a id="ref4" name="ref4"></a> Rosenbaum PR, Rubin DB. Reducing bias in observational studies using subclassification on the propensity score. <em>J Am Stat Assoc</em> 1984; 79:516-24.</li><li><a id="ref5" name="ref5"></a> Rubin DB. Estimating causal effects from large data sets using propensity scores. <em>Ann Intern Med</em> 1997; 127(8 Pt 2):757-63.</li><li><a id="ref6" name="ref6"></a> Rosenbaum PR, Rubin DB. Assessing sensitivity to an unobserved binary covariate in an observational study with binary outcome. <em>Journal of the Royal Statistical Society, Series B</em> 1983b; 45:212-218.</li><li><a id="ref7" name="ref7"></a> Angrist JD, Imbens GW, Rubin DB. Identification of causal effects using instrumental variables (with Discussions). <em>J Am Stat Assoc</em> 1996: 91(434):444-72.</li><li><a id="ref8" name="ref8"></a> Rosenbaum PR. Observational studies (2<sup>nd</sup> Edition). New York:Springer Verlag; 2002.</li></ol><hr /><p class="size2"><a id="notea" name="notea"></a><sup>a</sup> This strategy can be combined with pre-post comparisons, to be discussed in the following section. For now we assume that the pre-intervention outcome measures are not available.</p> </div><!-- end: Basic Modal -->
|
|
<div class="current-as-of">Page last reviewed December 2010</div>
|
|
<div class="citation">
|
|
<span>Internet Citation: Chapter 12. Description of Ideal Evaluation Methods: Quantitative Approaches to Context Heterogeneity: Assessing the Evidence for Context-Sensitive Effectiveness and Safety .
|
|
December 2010. Agency for Healthcare Research and Quality, Rockville, MD. http://archive.ahrq.gov/research/findings/final-reports/contextsensitive/context12.html</span>
|
|
<div class="citation-flag"></div>
|
|
</div>
|
|
</article>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<p> </p>
|
|
<div class="footnote">
|
|
<p> The information on this page is archived and provided for reference purposes only.</p>
|
|
</div>
|
|
<p> </p>
|
|
</div>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div><!-- End of Content Body -->
|
|
<!-- Footer graphic 1.2-->
|
|
<table border="0" cellSpacing="0" cellPadding="0" width="100%">
|
|
<tbody>
|
|
<tr>
|
|
<td background="/images/bottom_ahrq_bkg.jpg" width="125">
|
|
<img alt="AHRQ" src="/images/bottom_ahrq_1.jpg" width="125">
|
|
</td>
|
|
<td background="/images/bottom_ahrq_bkg.jpg" width="100%">
|
|
<img alt="" src="/images/bottom_ahrq_bkg.jpg" width="10" height="34">
|
|
</td>
|
|
<td background="/images/bottom_ahrq_bkg.jpg" width="310">
|
|
<img alt="Advancing Excellence in Health Care" src="/images/bottom_ahrq_2.gif" width="310" height="34">
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<!-- Footer links section -->
|
|
<div id="banner_Footer2">
|
|
<p>
|
|
<a href="https://www.ahrq.gov/">AHRQ Home</a> |
|
|
<a class="footer_navlink" href="https://info.ahrq.gov/cgi-bin/ahrq.cfg/php/enduser/std_alp.php">Questions?</a> |
|
|
<a class="footer_navlink" href="https://www.ahrq.gov/contact/index.html">Contact AHRQ</a> |
|
|
<a class="footer_navlink" href="https://www.ahrq.gov/policy/electronic/accessibility/index.html">Accessibility</a> |
|
|
<a class="footer_navlink" href="https://www.ahrq.gov/policy/electronic/privacy/index.html">Privacy Policy</a> |
|
|
<a class="footer_navlink" href="https://www.ahrq.gov/news/foia.htm">Freedom of Information Act</a> |
|
|
<a class="footer_navlink" href="https://www.ahrq.gov/policy/electronic/disclaimers/index.html">Disclaimers</a> |
|
|
<a class="footer_navlink" href="http://www.hhs.gov/open/recordsandreports/plainwritingact/index.html">Plain Writing Act</a>
|
|
<br/>
|
|
<a class="footer_navlink" href="http://www.hhs.gov">U.S. Department of Health & Human Services</a> |
|
|
<a class="footer_navlink" href="http://www.whitehouse.gov">The White House</a> |
|
|
<a class="footer_navlink" href="http://www.usa.gov">USA.gov: The U.S. Government's Official Web Portal</a>
|
|
</p>
|
|
</div>
|
|
<div id="banner_Footeraddress">
|
|
<p>
|
|
Agency for Healthcare Research and Quality <img alt="" src="/images/bottom_dot.gif"> 5600 Fishers Lane Rockville, MD 20857 <img alt="" src="/images/bottom_dot.gif">
|
|
Telephone: (301) 427-1364
|
|
</p>
|
|
</div>
|
|
<!-- End of Footer links section -->
|
|
|
|
<!-- JavaScript at the bottom for fast page loading: http://developer.yahoo.com/performance/rules.html#js_bottom -->
|
|
<!-- Grab Google CDN's jQuery, with a protocol relative URL; fall back to local if offline -->
|
|
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js"></script>
|
|
<script>window.jQuery || document.write('<script src="js/libs/jquery-1.7.2.min.js"><\/script>')</script>
|
|
|
|
<!-- scripts concatenated and minified via build script -->
|
|
<script src="/js/plugins.js"></script>
|
|
<script src="/js/script.js"></script>
|
|
<script src="/scripts/javascript.js"></script>
|
|
<!-- end scripts -->
|
|
</body>
|
|
</html>
|