490 lines
No EOL
61 KiB
HTML
490 lines
No EOL
61 KiB
HTML
<?xml version="1.0" encoding="utf-8"?>
|
||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
||
|
||
<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||
<!-- AppResources meta begin -->
|
||
<meta name="paf-app-resources" content="" />
|
||
<script type="text/javascript">var ncbi_startTime = new Date();</script>
|
||
|
||
<!-- AppResources meta end -->
|
||
|
||
<!-- TemplateResources meta begin -->
|
||
<meta name="paf_template" content="" />
|
||
|
||
<!-- TemplateResources meta end -->
|
||
|
||
<!-- Logger begin -->
|
||
<meta name="ncbi_db" content="books" /><meta name="ncbi_pdid" content="book-toc" /><meta name="ncbi_acc" content="NBK92293" /><meta name="ncbi_domain" content="methresvalid" /><meta name="ncbi_report" content="record" /><meta name="ncbi_type" content="fulltext" /><meta name="ncbi_objectid" content="" /><meta name="ncbi_pcid" content="/NBK92293/" /><meta name="ncbi_pagename" content="Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments - NCBI Bookshelf" /><meta name="ncbi_bookparttype" content="toc" /><meta name="ncbi_app" content="bookshelf" />
|
||
<!-- Logger end -->
|
||
|
||
<title>Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments - NCBI Bookshelf</title>
|
||
|
||
<!-- AppResources external_resources begin -->
|
||
<link rel="stylesheet" href="/core/jig/1.15.2/css/jig.min.css" /><script type="text/javascript" src="/core/jig/1.15.2/js/jig.min.js"></script>
|
||
|
||
<!-- AppResources external_resources end -->
|
||
|
||
<!-- Page meta begin -->
|
||
<meta name="robots" content="INDEX,FOLLOW,NOARCHIVE" /><meta name="author" content="Lisa Hartling, Michele Hamm, Andrea Milne, Ben Vandermeer, P Lina Santaguida, Mohammed Ansari, Alexander Tsertsvadze, Susanne Hempel, Paul Shekelle, Donna M Dryden" /><meta name="citation_title" content="Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments" /><meta name="citation_publisher" content="Agency for Healthcare Research and Quality (US)" /><meta name="citation_date" content="2012/03" /><meta name="citation_author" content="Lisa Hartling" /><meta name="citation_author" content="Michele Hamm" /><meta name="citation_author" content="Andrea Milne" /><meta name="citation_author" content="Ben Vandermeer" /><meta name="citation_author" content="P Lina Santaguida" /><meta name="citation_author" content="Mohammed Ansari" /><meta name="citation_author" content="Alexander Tsertsvadze" /><meta name="citation_author" content="Susanne Hempel" /><meta name="citation_author" content="Paul Shekelle" /><meta name="citation_author" content="Donna M Dryden" /><meta name="citation_pmid" content="22536612" /><meta name="citation_fulltext_html_url" content="https://www.ncbi.nlm.nih.gov/books/NBK92293/" /><link rel="schema.DC" href="http://purl.org/DC/elements/1.0/" /><meta name="DC.Title" content="Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments" /><meta name="DC.Type" content="Text" /><meta name="DC.Publisher" content="Agency for Healthcare Research and Quality (US)" /><meta name="DC.Contributor" content="Lisa Hartling" /><meta name="DC.Contributor" content="Michele Hamm" /><meta name="DC.Contributor" content="Andrea Milne" /><meta name="DC.Contributor" content="Ben Vandermeer" /><meta name="DC.Contributor" content="P Lina Santaguida" /><meta name="DC.Contributor" content="Mohammed Ansari" /><meta name="DC.Contributor" content="Alexander Tsertsvadze" /><meta name="DC.Contributor" content="Susanne Hempel" /><meta name="DC.Contributor" content="Paul Shekelle" /><meta name="DC.Contributor" content="Donna M Dryden" /><meta name="DC.Date" content="2012/03" /><meta name="DC.Identifier" content="https://www.ncbi.nlm.nih.gov/books/NBK92293/" /><meta name="DC.Language" content="en" /><meta name="description" content="Numerous tools exist to assess methodological quality, or risk of bias in systematic reviews; however, few have undergone extensive reliability or validity testing." /><meta name="og:title" content="Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments" /><meta name="og:type" content="book" /><meta name="og:description" content="Numerous tools exist to assess methodological quality, or risk of bias in systematic reviews; however, few have undergone extensive reliability or validity testing." /><meta name="og:url" content="https://www.ncbi.nlm.nih.gov/books/NBK92293/" /><meta name="og:site_name" content="NCBI Bookshelf" /><meta name="og:image" content="https://www.ncbi.nlm.nih.gov/corehtml/pmc/pmcgifs/bookshelf/thumbs/th-methresvalid-lrg.png" /><meta name="twitter:card" content="summary" /><meta name="twitter:site" content="@ncbibooks" /><meta name="bk-non-canon-loc" content="/books/n/methresvalid/toc/" /><link rel="canonical" href="https://www.ncbi.nlm.nih.gov/books/NBK92293/" /><link rel="stylesheet" href="/corehtml/pmc/css/figpopup.css" type="text/css" media="screen" /><link rel="stylesheet" href="/corehtml/pmc/css/bookshelf/2.26/css/books.min.css" type="text/css" /><link rel="stylesheet" href="/corehtml/pmc/css/bookshelf/2.26/css/books_print.min.css" type="text/css" media="print" /><style type="text/css">p a.figpopup{display:inline !important} .bk_tt {font-family: monospace} .first-line-outdent .bk_ref {display: inline} </style><script type="text/javascript" src="/corehtml/pmc/js/jquery.hoverIntent.min.js"> </script><script type="text/javascript" src="/corehtml/pmc/js/common.min.js?_=3.18"> </script><script type="text/javascript" src="/corehtml/pmc/js/large-obj-scrollbars.min.js"> </script><script type="text/javascript">window.name="mainwindow";</script><script type="text/javascript" src="/corehtml/pmc/js/bookshelf/2.26/book-toc.min.js"> </script><script type="text/javascript" src="/corehtml/pmc/js/bookshelf/2.26/books.min.js"> </script><meta name="book-collection" content="methodscollect" />
|
||
|
||
<!-- Page meta end -->
|
||
<link rel="shortcut icon" href="//www.ncbi.nlm.nih.gov/favicon.ico" /><meta name="ncbi_phid" content="CE8DABE57D69EB310000000001270101.m_13" />
|
||
<meta name='referrer' content='origin-when-cross-origin'/><link type="text/css" rel="stylesheet" href="//static.pubmed.gov/portal/portal3rc.fcgi/4216699/css/3852956/3985586/3808861/4121862/3974050/3917732/251717/4216701/14534/45193/4113719/3849091/3984811/3751656/4033350/3840896/3577051/3852958/4008682/4207974/4206132/4062871/12930/3964959/3854974/36029/4128070/9685/3549676/3609192/3609193/3609213/3395586.css" /><link type="text/css" rel="stylesheet" href="//static.pubmed.gov/portal/portal3rc.fcgi/4216699/css/3411343/3882866.css" media="print" /></head>
|
||
<body class="book-toc">
|
||
<div class="grid">
|
||
<div class="col twelve_col nomargin shadow">
|
||
<!-- System messages like service outage or JS required; this is handled by the TemplateResources portlet -->
|
||
<div class="sysmessages">
|
||
<noscript>
|
||
<p class="nojs">
|
||
<strong>Warning:</strong>
|
||
The NCBI web site requires JavaScript to function.
|
||
<a href="/guide/browsers/#enablejs" title="Learn how to enable JavaScript" target="_blank">more...</a>
|
||
</p>
|
||
</noscript>
|
||
</div>
|
||
<!--/.sysmessage-->
|
||
<div class="wrap">
|
||
<div class="page">
|
||
<div class="top">
|
||
<div id="universal_header">
|
||
<section class="usa-banner">
|
||
<div class="usa-accordion">
|
||
<header class="usa-banner-header">
|
||
<div class="usa-grid usa-banner-inner">
|
||
<img src="https://www.ncbi.nlm.nih.gov/coreutils/uswds/img/favicons/favicon-57.png" alt="U.S. flag" />
|
||
<p>An official website of the United States government</p>
|
||
<button class="non-usa-accordion-button usa-banner-button" aria-expanded="false" aria-controls="gov-banner-top" type="button">
|
||
<span class="usa-banner-button-text">Here's how you know</span>
|
||
</button>
|
||
</div>
|
||
</header>
|
||
<div class="usa-banner-content usa-grid usa-accordion-content" id="gov-banner-top" aria-hidden="true">
|
||
<div class="usa-banner-guidance-gov usa-width-one-half">
|
||
<img class="usa-banner-icon usa-media_block-img" src="https://www.ncbi.nlm.nih.gov/coreutils/uswds/img/icon-dot-gov.svg" alt="Dot gov" />
|
||
<div class="usa-media_block-body">
|
||
<p>
|
||
<strong>The .gov means it's official.</strong>
|
||
<br />
|
||
Federal government websites often end in .gov or .mil. Before
|
||
sharing sensitive information, make sure you're on a federal
|
||
government site.
|
||
</p>
|
||
</div>
|
||
</div>
|
||
<div class="usa-banner-guidance-ssl usa-width-one-half">
|
||
<img class="usa-banner-icon usa-media_block-img" src="https://www.ncbi.nlm.nih.gov/coreutils/uswds/img/icon-https.svg" alt="Https" />
|
||
<div class="usa-media_block-body">
|
||
<p>
|
||
<strong>The site is secure.</strong>
|
||
<br />
|
||
The <strong>https://</strong> ensures that you are connecting to the
|
||
official website and that any information you provide is encrypted
|
||
and transmitted securely.
|
||
</p>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<div class="usa-overlay"></div>
|
||
<header class="ncbi-header" role="banner" data-section="Header">
|
||
|
||
<div class="usa-grid">
|
||
<div class="usa-width-one-whole">
|
||
|
||
<div class="ncbi-header__logo">
|
||
<a href="/" class="logo" aria-label="NCBI Logo" data-ga-action="click_image" data-ga-label="NIH NLM Logo">
|
||
<img src="https://www.ncbi.nlm.nih.gov/coreutils/nwds/img/logos/AgencyLogo.svg" alt="NIH NLM Logo" />
|
||
</a>
|
||
</div>
|
||
|
||
<div class="ncbi-header__account">
|
||
<a id="account_login" href="https://account.ncbi.nlm.nih.gov" class="usa-button header-button" style="display:none" data-ga-action="open_menu" data-ga-label="account_menu">Log in</a>
|
||
<button id="account_info" class="header-button" style="display:none" aria-controls="account_popup" type="button">
|
||
<span class="fa fa-user" aria-hidden="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="20px" height="20px">
|
||
<g style="fill: #fff">
|
||
<ellipse cx="12" cy="8" rx="5" ry="6"></ellipse>
|
||
<path d="M21.8,19.1c-0.9-1.8-2.6-3.3-4.8-4.2c-0.6-0.2-1.3-0.2-1.8,0.1c-1,0.6-2,0.9-3.2,0.9s-2.2-0.3-3.2-0.9 C8.3,14.8,7.6,14.7,7,15c-2.2,0.9-3.9,2.4-4.8,4.2C1.5,20.5,2.6,22,4.1,22h15.8C21.4,22,22.5,20.5,21.8,19.1z"></path>
|
||
</g>
|
||
</svg>
|
||
</span>
|
||
<span class="username desktop-only" aria-hidden="true" id="uname_short"></span>
|
||
<span class="sr-only">Show account info</span>
|
||
</button>
|
||
</div>
|
||
|
||
<div class="ncbi-popup-anchor">
|
||
<div class="ncbi-popup account-popup" id="account_popup" aria-hidden="true">
|
||
<div class="ncbi-popup-head">
|
||
<button class="ncbi-close-button" data-ga-action="close_menu" data-ga-label="account_menu" type="button">
|
||
<span class="fa fa-times">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48" width="24px" height="24px">
|
||
<path d="M38 12.83l-2.83-2.83-11.17 11.17-11.17-11.17-2.83 2.83 11.17 11.17-11.17 11.17 2.83 2.83 11.17-11.17 11.17 11.17 2.83-2.83-11.17-11.17z"></path>
|
||
</svg>
|
||
</span>
|
||
<span class="usa-sr-only">Close</span></button>
|
||
<h4>Account</h4>
|
||
</div>
|
||
<div class="account-user-info">
|
||
Logged in as:<br />
|
||
<b><span class="username" id="uname_long">username</span></b>
|
||
</div>
|
||
<div class="account-links">
|
||
<ul class="usa-unstyled-list">
|
||
<li><a id="account_myncbi" href="/myncbi/" class="set-base-url" data-ga-action="click_menu_item" data-ga-label="account_myncbi">Dashboard</a></li>
|
||
<li><a id="account_pubs" href="/myncbi/collections/bibliography/" class="set-base-url" data-ga-action="click_menu_item" data-ga-label="account_pubs">Publications</a></li>
|
||
<li><a id="account_settings" href="/account/settings/" class="set-base-url" data-ga-action="click_menu_item" data-ga-label="account_settings">Account settings</a></li>
|
||
<li><a id="account_logout" href="/account/signout/" class="set-base-url" data-ga-action="click_menu_item" data-ga-label="account_logout">Log out</a></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
</header>
|
||
<div role="navigation" aria-label="access keys">
|
||
<a id="nws_header_accesskey_0" href="https://www.ncbi.nlm.nih.gov/guide/browsers/#ncbi_accesskeys" class="usa-sr-only" accesskey="0" tabindex="-1">Access keys</a>
|
||
<a id="nws_header_accesskey_1" href="https://www.ncbi.nlm.nih.gov" class="usa-sr-only" accesskey="1" tabindex="-1">NCBI Homepage</a>
|
||
<a id="nws_header_accesskey_2" href="/myncbi/" class="set-base-url usa-sr-only" accesskey="2" tabindex="-1">MyNCBI Homepage</a>
|
||
<a id="nws_header_accesskey_3" href="#maincontent" class="usa-sr-only" accesskey="3" tabindex="-1">Main Content</a>
|
||
<a id="nws_header_accesskey_4" href="#" class="usa-sr-only" accesskey="4" tabindex="-1">Main Navigation</a>
|
||
</div>
|
||
<section data-section="Alerts">
|
||
<div class="ncbi-alerts-placeholder"></div>
|
||
</section>
|
||
</div>
|
||
<div class="header">
|
||
<div class="res_logo"><h1 class="res_name"><a href="/books/" title="Bookshelf home">Bookshelf</a></h1><h2 class="res_tagline"></h2></div>
|
||
<div class="search"><form method="get" action="/books/"><div class="search_form"><label for="database" class="offscreen_noflow">Search database</label><select id="database"><optgroup label="Recent"><option value="books" selected="selected" data-ac_dict="bookshelf-search">Books</option><option value="pubmed">PubMed</option><option value="nlmcatalog">NLM Catalog</option><option value="nuccore" class="last">Nucleotide</option></optgroup><optgroup label="All"><option value="gquery">All Databases</option><option value="assembly">Assembly</option><option value="biocollections">Biocollections</option><option value="bioproject">BioProject</option><option value="biosample">BioSample</option><option value="books" data-ac_dict="bookshelf-search">Books</option><option value="clinvar">ClinVar</option><option value="cdd">Conserved Domains</option><option value="gap">dbGaP</option><option value="dbvar">dbVar</option><option value="gene">Gene</option><option value="genome">Genome</option><option value="gds">GEO DataSets</option><option value="geoprofiles">GEO Profiles</option><option value="gtr">GTR</option><option value="ipg">Identical Protein Groups</option><option value="medgen">MedGen</option><option value="mesh">MeSH</option><option value="nlmcatalog">NLM Catalog</option><option value="nuccore">Nucleotide</option><option value="omim">OMIM</option><option value="pmc">PMC</option><option value="protein">Protein</option><option value="proteinclusters">Protein Clusters</option><option value="protfam">Protein Family Models</option><option value="pcassay">PubChem BioAssay</option><option value="pccompound">PubChem Compound</option><option value="pcsubstance">PubChem Substance</option><option value="pubmed">PubMed</option><option value="snp">SNP</option><option value="sra">SRA</option><option value="structure">Structure</option><option value="taxonomy">Taxonomy</option><option value="toolkit">ToolKit</option><option value="toolkitall">ToolKitAll</option><option value="toolkitbookgh">ToolKitBookgh</option></optgroup></select><div class="nowrap"><label for="term" class="offscreen_noflow" accesskey="/">Search term</label><div class="nowrap"><input type="text" name="term" id="term" title="Search Books. Use up and down arrows to choose an item from the autocomplete." value="" class="jig-ncbiclearbutton jig-ncbiautocomplete" data-jigconfig="dictionary:'bookshelf-search',disableUrl:'NcbiSearchBarAutoComplCtrl'" autocomplete="off" data-sbconfig="ds:'no',pjs:'no',afs:'no'" /></div><button id="search" type="submit" class="button_search nowrap" cmd="go">Search</button></div></div></form><ul class="searchlinks inline_list"><li>
|
||
<a href="/books/browse/">Browse Titles</a>
|
||
</li><li>
|
||
<a href="/books/advanced/">Advanced</a>
|
||
</li><li class="help">
|
||
<a href="/books/NBK3833/">Help</a>
|
||
</li><li class="disclaimer">
|
||
<a target="_blank" data-ga-category="literature_resources" data-ga-action="link_click" data-ga-label="disclaimer_link" href="https://www.ncbi.nlm.nih.gov/books/about/disclaimer/">Disclaimer</a>
|
||
</li></ul></div>
|
||
</div>
|
||
|
||
|
||
|
||
<!--<component id="Page" label="headcontent"/>-->
|
||
|
||
</div>
|
||
<div class="content">
|
||
<!-- site messages -->
|
||
<!-- Custom content 1 -->
|
||
<div class="col1">
|
||
|
||
</div>
|
||
|
||
<div class="container">
|
||
<div id="maincontent" class="content eight_col col">
|
||
<!-- Custom content in the left column above book nav -->
|
||
<div class="col2">
|
||
|
||
</div>
|
||
|
||
<!-- Book content -->
|
||
|
||
|
||
<!-- Custom content between navigation and content -->
|
||
<div class="col3">
|
||
|
||
</div>
|
||
|
||
<div class="source">
|
||
<div class="pre-content"><div><div class="bk_prnt"><p class="small">NCBI Bookshelf. A service of the National Library of Medicine, National Institutes of Health.</p></div><div class="pagination bk_noprnt"><span class="inactive page_link prev">< Prev</span><a class="active page_link next" href="/books/n/methresvalid/fm.s1/" title="Next page in this title">Next ></a></div></div></div>
|
||
<div class="main-content lit-style" itemscope="itemscope" itemtype="http://schema.org/Book"><div class="meta-content fm-sec"><div class="iconblock whole_rhythm clearfix no_top_margin"><a href="https://effectivehealthcare.ahrq.gov" title="Agency for Healthcare Research and Quality (US)" class="img_link icnblk_img" ref="pagearea=logo&targetsite=external&targetcat=link&targettype=publisher"><img class="source-thumb" src="/corehtml/pmc/pmcgifs/bookshelf/thumbs/th-methresvalid-lrg.png" alt="Cover of Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments" /></a><div class="icnblk_cntnt"><h1 id="_NBK92293_"><span itemprop="name">Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments</span></h1><p><i>Methods Research Reports</i></p><p class="contrib-group">Investigators: <span itemprop="author">Lisa Hartling</span>, PhD, <span itemprop="author">Michele Hamm</span>, MSc, <span itemprop="author">Andrea Milne</span>, MLIS, <span itemprop="author">Ben Vandermeer</span>, MSc, <span itemprop="author">P Lina Santaguida</span>, PhD, <span itemprop="author">Mohammed Ansari</span>, MBBS, MMedSc, MPhil, <span itemprop="author">Alexander Tsertsvadze</span>, MD, MSc, <span itemprop="author">Susanne Hempel</span>, PhD, <span itemprop="author">Paul Shekelle</span>, MD, PhD, and <span itemprop="author">Donna M Dryden</span>, PhD.</p><a data-jig="ncbitoggler" href="#__NBK92293_ai__" style="border:0;text-decoration:none">Author Information and Affiliations</a><div style="display:none" class="ui-widget" id="__NBK92293_ai__"><p class="contrib-group"><h4>Authors</h4>Investigators: <span itemprop="author">Lisa Hartling</span>, PhD, <span itemprop="author">Michele Hamm</span>, MSc, <span itemprop="author">Andrea Milne</span>, MLIS, <span itemprop="author">Ben Vandermeer</span>, MSc, <span itemprop="author">P Lina Santaguida</span>, PhD, <span itemprop="author">Mohammed Ansari</span>, MBBS, MMedSc, MPhil, <span itemprop="author">Alexander Tsertsvadze</span>, MD, MSc, <span itemprop="author">Susanne Hempel</span>, PhD, <span itemprop="author">Paul Shekelle</span>, MD, PhD, and <span itemprop="author">Donna M Dryden</span>, PhD.</p><h4>Affiliations</h4><div class="affiliation"><sup>1</sup> University of Alberta Evidence-based Practice Center</div></div><div class="half_rhythm">Rockville (MD): <a href="https://effectivehealthcare.ahrq.gov" ref="pagearea=meta&targetsite=external&targetcat=link&targettype=publisher"><span itemprop="publisher">Agency for Healthcare Research and Quality (US)</span></a>; <span itemprop="datePublished">2012 Mar</span>.<div class="small">Report No.: 12-EHC039-EF</div></div><div class="half_rhythm"><ul class="inline_list"><li><span class="label"><a data-jig="ncbidialog" href="#_ncbi_dlg_cpyrght_NBK92293" data-jigconfig="modal:true">Copyright and
|
||
Permissions</a></span></li></ul></div><div id="_ncbi_dlg_cpyrght_NBK92293" style="display:none" title="Copyright and Permissions"><div>This publication is in the public domain. For more information, see the <a href="/books/about/copyright/">Bookshelf Copyright Notice</a>.</div></div><div class="bk_noprnt"><form method="get" action="/books/n/methresvalid/" id="bk_srch"><div class="bk_search"><label for="bk_term" class="offscreen_noflow">Search term</label><input type="text" title="Search this book" id="bk_term" name="term" value="" data-jig="ncbiclearbutton" /> <input type="submit" class="jig-ncbibutton" value="Search this book" submit="false" style="padding: 0.1em 0.4em;" /></div></form></div></div></div></div><div class="body-content whole_rhythm" itemprop="text"><div itemprop="description"><h2>Structured Abstract</h2><div id="background"><h4 class="inline">Background:</h4><p>Numerous tools exist to assess methodological quality, or risk of bias in systematic reviews; however, few have undergone extensive reliability or validity testing.</p></div><div id="objectives"><h4 class="inline">Objectives:</h4><p>(1) assess the reliability of the Cochrane Risk of Bias (ROB) tool for randomized controlled trials (RCTs) and the Newcastle-Ottawa Scale (NOS) for cohort studies between individual raters, and between consensus agreements of individual raters for the ROB tool; (2) assess the validity of the Cochrane ROB tool and NOS by examining the association between study quality and treatment effect size (ES); (3) examine the impact of study-level factors on reliability and validity.</p></div><div id="method"><h4 class="inline">Methods:</h4><p>Two reviewers independently assessed risk of bias for 154 RCTs. For a subset of 30 RCTs, two reviewers from each of four Evidence-based Practice Centers assessed risk of bias and reached consensus. Inter-rater agreement was assessed using kappa statistics. We assessed the association between ES and risk of bias using meta-regression. We examined the impact of study-level factors on the association between risk of bias and ES using subgroup analyses. Two reviewers independently applied the NOS to 131 cohort studies from 8 meta-analyses. Inter-rater agreement was calculated using kappa statistics. Within each meta-analysis, we generated a ratio of pooled estimates for each quality domain. The ratios were combined to give an overall estimate of differences in effect estimates with inverse-variance weighting and a random effects model.</p></div><div id="result"><h4 class="inline">Results:</h4><p>Inter-rater reliability between two reviewers was considered fair for most domains (κ ranging from 0.24 to 0.37), except for sequence generation (κ=0.79, substantial). Inter-rater reliability of consensus assessments across four reviewer pairs was moderate for sequence generation (κ=0.60), fair for allocation concealment and “other sources of bias” (κ=0.37, 0.27), and slight for the remaining domains (κ ranging from 0.05 to 0.09). Inter-rater variability was influenced by study-level factors including nature of outcome, nature of intervention, study design, trial hypothesis, and funding source. Inter-rater variability resulted more often from different interpretation of the tool rather than different information identified in the study reports. No statistically significant differences were found in ES when comparing studies categorized as high, unclear or low risk of bias. Inter-rater reliability of the NOS varied from substantial for length of followup to poor for selection of non-exposed cohort and demonstration that the outcome was not present at outset of study. We found no association between individual NOS items or overall NOS score and effect estimates.</p></div><div id="conclusion"><h4 class="inline">Conclusion:</h4><p>More specific guidance is needed to apply risk of bias/quality tools. Study-level factors that were shown to influence agreement provide direction for detailed guidance. Low agreement across pairs of reviewers has implications for incorporation of risk of bias into results and grading the strength of evidence. Variable agreement for the NOS, and lack of evidence that it discriminates studies that may provide biased results, underscores the need for more detailed guidance to apply the tool in systematic reviews.</p></div></div><div><h2>Contents</h2><div class="bktoc_all_cntnr top align_right" style="display:none"><ul class="inline_list_right"><li><a class="bktoc_all_exp" href="#">Expand All</a></li><li style="margin-left:.8em"><a class="bktoc_all_clps" href="#">Collapse All</a></li></ul></div><ul id="toc_tllNBK92293_fm_s1" class="simple-list toc toc-toggle"><li class="half_rhythm" id="toc_itm_NBK92293_fm_s1"><a href="/books/n/methresvalid/fm.s1/" class="toc-item">Preface</a></li><li class="half_rhythm" id="toc_itm_NBK92293_ack"><a href="/books/n/methresvalid/ack/" class="toc-item">Acknowledgments</a></li><li class="half_rhythm" id="toc_itm_NBK92293_fm_s2"><a href="/books/n/methresvalid/fm.s2/" class="toc-item">Steering Committee</a></li><li class="half_rhythm" id="toc_itm_NBK92293_fm_s3"><a href="/books/n/methresvalid/fm.s3/" class="toc-item">Peer Reviewers</a></li><li class="half_rhythm" id="toc_itm_NBK92293_executivesummary"><a href="/books/n/methresvalid/executivesummary/" class="toc-item">Executive Summary</a></li><li class="half_rhythm" id="toc_itm_NBK92293_introduction"><a href="/books/n/methresvalid/introduction/" class="toc-item">Introduction</a><ul id="toc_lst_NBK92293_introduction_s1" class="simple-list toc bktoc_lst_exp"><li class="half_rhythm" id="toc_itm_NBK92293_introduction_s1"><a href="/books/n/methresvalid/introduction/#introduction.s1" class="toc-item">Quality and Risk of Bias Assessment in Systematic Reviews</a></li><li class="half_rhythm" id="toc_itm_NBK92293_introduction_s4"><a href="/books/n/methresvalid/introduction/#introduction.s4" class="toc-item">Goal and Objective</a></li></ul></li><li class="half_rhythm" id="toc_itm_NBK92293_methods"><a href="/books/n/methresvalid/methods/" class="toc-item">Methods</a><ul id="toc_lst_NBK92293_methods_s1" class="simple-list toc bktoc_lst_exp"><li class="half_rhythm" id="toc_itm_NBK92293_methods_s1"><a href="/books/n/methresvalid/methods/#methods.s1" class="toc-item">Steering Committee</a></li><li class="half_rhythm" id="toc_itm_NBK92293_methods_s2"><a href="/books/n/methresvalid/methods/#methods.s2" class="toc-item">General Approach</a></li><li class="half_rhythm" id="toc_itm_NBK92293_methods_s3"><a href="/books/n/methresvalid/methods/#methods.s3" class="toc-item">Risk of Bias and Randomized Controlled Trials</a></li><li class="half_rhythm" id="toc_itm_NBK92293_methods_s8"><a href="/books/n/methresvalid/methods/#methods.s8" class="toc-item">Newcastle-Ottawa Scale and Cohort Studies</a></li></ul></li><li class="half_rhythm" id="toc_itm_NBK92293_results"><a href="/books/n/methresvalid/results/" class="toc-item">Results</a><ul id="toc_lst_NBK92293_results_s1" class="simple-list toc bktoc_lst_exp"><li class="half_rhythm" id="toc_itm_NBK92293_results_s1"><a href="/books/n/methresvalid/results/#results.s1" class="toc-item">Overview</a></li><li class="half_rhythm" id="toc_itm_NBK92293_results_s2"><a href="/books/n/methresvalid/results/#results.s2" class="toc-item">Risk of Bias and Randomized Controlled Trials</a></li><li class="half_rhythm" id="toc_itm_NBK92293_results_s8"><a href="/books/n/methresvalid/results/#results.s8" class="toc-item">Newcastle-Ottawa Scale and Cohort Studies</a></li></ul></li><li class="half_rhythm" id="toc_itm_NBK92293_summary"><a href="/books/n/methresvalid/summary/" class="toc-item">Summary and Discussion</a><ul id="toc_lst_NBK92293_summary_s1" class="simple-list toc bktoc_lst_exp"><li class="half_rhythm" id="toc_itm_NBK92293_summary_s1"><a href="/books/n/methresvalid/summary/#summary.s1" class="toc-item">Key Points</a></li><li class="half_rhythm" id="toc_itm_NBK92293_summary_s4"><a href="/books/n/methresvalid/summary/#summary.s4" class="toc-item">Discussion</a></li><li class="half_rhythm" id="toc_itm_NBK92293_summary_s7"><a href="/books/n/methresvalid/summary/#summary.s7" class="toc-item">Implications for Practice</a></li><li class="half_rhythm" id="toc_itm_NBK92293_summary_s8"><a href="/books/n/methresvalid/summary/#summary.s8" class="toc-item">Future Directions</a></li><li class="half_rhythm" id="toc_itm_NBK92293_summary_s9"><a href="/books/n/methresvalid/summary/#summary.s9" class="toc-item">Strengths and Limitations</a></li><li class="half_rhythm" id="toc_itm_NBK92293_summary_s10"><a href="/books/n/methresvalid/summary/#summary.s10" class="toc-item">Conclusions</a></li></ul></li><li class="half_rhythm" id="toc_itm_NBK92293_references_rl1"><a href="/books/n/methresvalid/references.rl1/" class="toc-item">References</a></li><li class="half_rhythm" id="toc_itm_NBK92293_abbreviations"><a href="/books/n/methresvalid/abbreviations/" class="toc-item">Abbreviations</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_appgroup1"><a href="/books/n/methresvalid/appendixes.appgroup1/" class="toc-item">Appendixes</a><ul id="toc_lst_NBK92293_appendixes_app1" class="simple-list toc bktoc_lst_exp"><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app1"><a href="/books/n/methresvalid/appendixes.app1/" class="toc-item">Appendix A Sample of Randomized Controlled Trials</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app2"><a href="/books/n/methresvalid/appendixes.app2/" class="toc-item">Appendix B Guidelines for Risk of Bias Assessments</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app3"><a href="/books/n/methresvalid/appendixes.app3/" class="toc-item">Appendix C Variables for Data Extraction from Randomized Controlled Trials</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app4"><a href="/books/n/methresvalid/appendixes.app4/" class="toc-item">Appendix D Meta-Analyses and Cohort Studies Used for NOS Assessments</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app5"><a href="/books/n/methresvalid/appendixes.app5/" class="toc-item">Appendix E Decision Rules for Application of the Newcastle-Ottawa Scale</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app6"><a href="/books/n/methresvalid/appendixes.app6/" class="toc-item">Appendix F Supplementary Information for NOS Assessments</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app7"><a href="/books/n/methresvalid/appendixes.app7/" class="toc-item">Appendix G Description of Randomized Controlled Trials</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app8"><a href="/books/n/methresvalid/appendixes.app8/" class="toc-item">Appendix H Data Used for Kappa Calculations</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app9"><a href="/books/n/methresvalid/appendixes.app9/" class="toc-item">Appendix I Inter-rater Reliability on Risk of Bias Assessments, by Domain and Study-level Variable With Confidence Intervals</a></li><li class="half_rhythm" id="toc_itm_NBK92293_appendixes_app10"><a href="/books/n/methresvalid/appendixes.app10/" class="toc-item">Appendix J Sources of Disagreement Across Consensus Assessments Using the Risk of Bias Tool</a></li></ul></li></ul><div class="bktoc_all_cntnr align_right" style="display:none"><ul class="inline_list_right"><li><a class="bktoc_all_exp" href="#">Expand All</a></li><li style="margin-left:.8em"><a class="bktoc_all_clps" href="#">Collapse All</a></li></ul></div></div><div><p><b>Prepared for:</b>
|
||
Agency for Healthcare Research and Quality, U.S. Department of Health and Human Services<a class="bk_pop" href="#address1"><sup>1</sup></a>, . <b>Contract No.</b>
|
||
290-2007-10021-I. <b>Prepared by:</b>
|
||
University of Alberta Evidence-based Practice Center, Edmonton, Alberta, Canada</p></div><div><h4 class="inline">Suggested citation:</h4><p>Hartling L, Hamm M, Milne A, Vandermeer B, Santaguida PL, Ansari M, Tsertsvadze A, Hempel S, Shekelle P, Dryden DM. Validity and inter-rater reliability testing of quality assessment instruments. (Prepared by the University of Alberta Evidence-based Practice Center under Contract No. 290-2007-10021-I.) AHRQ Publication No. 12-EHC039-EF. Rockville, MD: Agency for Healthcare Research and Quality. March 2012. <a href="http://www.effectivehealthcare.ahrq.gov/reports/final.cfm" ref="pagearea=body&targetsite=external&targetcat=link&targettype=uri">www.effectivehealthcare.ahrq.gov/reports/final.cfm</a>.</p></div><div><p>This report is based on research conducted by the University of Alberta Evidence-based Practice Center under contract to the Agency for Healthcare Research and Quality (AHRQ), Rockville, MD (Contract No. 290-2007-10021-1). The findings and conclusions in this document are those of the author(s), who are responsible for its content, and do not necessarily represent the views of AHRQ. No statement in this report should be construed as an official position of AHRQ or of the U.S. Department of Health and Human Services.</p><p>The information in this report is intended to help clinicians, employers, policymakers, and others make informed decisions about the provision of health care services. This report is intended as a reference and not as a substitute for clinical judgment.</p><p>This report may be used, in whole or in part, as the basis for the development of clinical practice guidelines and other quality enhancement tools, or as a basis for reimbursement and coverage policies. AHRQ or U.S. Department of Health and Human Services endorsement of such derivative products or actions may not be stated or implied.</p><p>None of the investigators have any affiliations or financial involvement that conflicts with the material presented in this report.</p></div><div><dl class="temp-labeled-list small"><dt>1</dt><dd><div id="address1"><p class="no_top_margin">540 Gaither Road, Rockville, MD 20850; <a href="http://www.ahrq.gov" ref="pagearea=body&targetsite=external&targetcat=link&targettype=uri">www<wbr style="display:inline-block"></wbr>.ahrq.gov</a></p></div></dd></dl></div></div></div>
|
||
<div class="post-content"><div><div class="half_rhythm"><a href="/books/about/copyright/">Copyright Notice</a></div><div class="small"><span class="label">Bookshelf ID: NBK92293</span><span class="label">PMID: <a href="https://pubmed.ncbi.nlm.nih.gov/22536612" title="PubMed record of this title" ref="pagearea=meta&targetsite=entrez&targetcat=link&targettype=pubmed">22536612</a></span></div><div style="margin-top:2em" class="bk_noprnt"><div class="pagination bk_noprnt"><span class="inactive page_link prev">< Prev</span><a class="active page_link next" href="/books/n/methresvalid/fm.s1/" title="Next page in this title">Next ></a></div></div></div></div>
|
||
|
||
</div>
|
||
|
||
<!-- Custom content below content -->
|
||
<div class="col4">
|
||
|
||
</div>
|
||
|
||
|
||
<!-- Book content -->
|
||
|
||
<!-- Custom contetnt below bottom nav -->
|
||
<div class="col5">
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div id="rightcolumn" class="four_col col last">
|
||
<!-- Custom content above discovery portlets -->
|
||
<div class="col6">
|
||
<div id="ncbi_share_book"><a href="#" class="ncbi_share" data-ncbi_share_config="popup:false,shorten:true" ref="id=NBK92293&db=books">Share</a></div>
|
||
|
||
</div>
|
||
<div xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"></div><div class="portlet"><div class="portlet_head"><div class="portlet_title"><h3><span>Views</span></h3></div><a name="Shutter" sid="1" href="#" class="portlet_shutter" title="Show/hide content" remembercollapsed="true" pgsec_name="PDF_download" id="Shutter"></a></div><div class="portlet_content"><ul xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="simple-list"><li><a href="/books/NBK92293/?report=reader">PubReader</a></li><li><a href="/books/NBK92293/?report=printable">Print View</a></li><li><a data-jig="ncbidialog" href="#_ncbi_dlg_citbx_NBK92293" data-jigconfig="width:400,modal:true">Cite this Page</a><div id="_ncbi_dlg_citbx_NBK92293" style="display:none" title="Cite this Page"><div class="bk_tt">Hartling L, Hamm M, Milne A, et al. Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments [Internet]. Rockville (MD): Agency for Healthcare Research and Quality (US); 2012 Mar. <span class="bk_cite_avail"></span></div></div></li><li><a href="/books/n/methresvalid/pdf/">PDF version of this title</a> (1.2M)</li></ul></div></div><div class="portlet"><div class="portlet_head"><div class="portlet_title"><h3><span>Other titles in these collections</span></h3></div><a name="Shutter" sid="1" href="#" class="portlet_shutter" title="Show/hide content" remembercollapsed="true" pgsec_name="source-links" id="Shutter"></a></div><div class="portlet_content"><ul xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="simple-list"><li><a href="/books/n/methodscollect/">AHRQ Methods for Effective Health Care</a></li><li><a href="/books/n/hstatcollect/">Health Services/Technology Assessment Texts
|
||
(HSTAT)</a></li></ul></div></div><div class="portlet"><div class="portlet_head"><div class="portlet_title"><h3><span>Related information</span></h3></div><a name="Shutter" sid="1" href="#" class="portlet_shutter" title="Show/hide content" remembercollapsed="true" pgsec_name="discovery_db_links" id="Shutter"></a></div><div class="portlet_content"><ul><li class="brieflinkpopper"><a class="brieflinkpopperctrl" href="/books/?Db=nlmcatalog&DbFrom=books&Cmd=Link&LinkName=books_nlmcatalog&IdsFromResult=2894794" ref="log$=recordlinks">NLM Catalog</a><div class="brieflinkpop offscreen_noflow">Related NLM Catalog Entries</div></li></ul></div></div><div class="portlet"><div class="portlet_head"><div class="portlet_title"><h3><span>Similar articles in PubMed</span></h3></div><a name="Shutter" sid="1" href="#" class="portlet_shutter" title="Show/hide content" remembercollapsed="true" pgsec_name="PBooksDiscovery_RA" id="Shutter"></a></div><div class="portlet_content"><ul><li class="brieflinkpopper two_line"><a class="brieflinkpopperctrl" href="/pubmed/22981249" ref="ordinalpos=1&linkpos=1&log$=relatedarticles&logdbfrom=pubmed">Testing the risk of bias tool showed low reliability between individual reviewers and across consensus assessments of reviewer pairs.</a><span class="source">[J Clin Epidemiol. 2013]</span><div class="brieflinkpop offscreen_noflow">Testing the risk of bias tool showed low reliability between individual reviewers and across consensus assessments of reviewer pairs.<div class="brieflinkpopdesc"><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="author">Hartling L, Hamm MP, Milne A, Vandermeer B, Santaguida PL, Ansari M, Tsertsvadze A, Hempel S, Shekelle P, Dryden DM. </em><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="cit">J Clin Epidemiol. 2013 Sep; 66(9):973-81. Epub 2012 Sep 13.</em></div></div></li><li class="brieflinkpopper two_line"><a class="brieflinkpopperctrl" href="/pubmed/23683848" ref="ordinalpos=1&linkpos=2&log$=relatedarticles&logdbfrom=pubmed">Testing the Newcastle Ottawa Scale showed low reliability between individual reviewers.</a><span class="source">[J Clin Epidemiol. 2013]</span><div class="brieflinkpop offscreen_noflow">Testing the Newcastle Ottawa Scale showed low reliability between individual reviewers.<div class="brieflinkpopdesc"><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="author">Hartling L, Milne A, Hamm MP, Vandermeer B, Ansari M, Tsertsvadze A, Dryden DM. </em><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="cit">J Clin Epidemiol. 2013 Sep; 66(9):982-93. Epub 2013 May 16.</em></div></div></li><li class="brieflinkpopper two_line"><a class="brieflinkpopperctrl" href="/pubmed/22764383" ref="ordinalpos=1&linkpos=3&log$=relatedreviews&logdbfrom=pubmed"><span xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="invert">Review</span> Reliability Testing of the AHRQ EPC Approach to Grading the Strength of Evidence in Comparative Effectiveness Reviews</a><span class="source">[ 2012]</span><div class="brieflinkpop offscreen_noflow"><span xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="invert">Review</span> Reliability Testing of the AHRQ EPC Approach to Grading the Strength of Evidence in Comparative Effectiveness Reviews<div class="brieflinkpopdesc"><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="author">Berkman ND, Lohr KN, Morgan LC, Richmond E, Kuo TM, Morton S, Viswanathan M, Kamerow D, West S, Tant E. </em><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="cit">2012 May</em></div></div></li><li class="brieflinkpopper two_line"><a class="brieflinkpopperctrl" href="/pubmed/32051035" ref="ordinalpos=1&linkpos=4&log$=relatedarticles&logdbfrom=pubmed">Inter-rater reliability and validity of risk of bias instrument for non-randomized studies of exposures: a study protocol.</a><span class="source">[Syst Rev. 2020]</span><div class="brieflinkpop offscreen_noflow">Inter-rater reliability and validity of risk of bias instrument for non-randomized studies of exposures: a study protocol.<div class="brieflinkpopdesc"><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="author">Jeyaraman MM, Al-Yousif N, Robson RC, Copstein L, Balijepalli C, Hofer K, Fazeli MS, Ansari MT, Tricco AC, Rabbani R, et al. </em><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="cit">Syst Rev. 2020 Feb 12; 9(1):32. Epub 2020 Feb 12.</em></div></div></li><li class="brieflinkpopper two_line"><a class="brieflinkpopperctrl" href="/pubmed/21390219" ref="ordinalpos=1&linkpos=5&log$=relatedreviews&logdbfrom=pubmed"><span xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="invert">Review</span> Applying the risk of bias tool in a systematic review of combination long-acting beta-agonists and inhaled corticosteroids for persistent asthma.</a><span class="source">[PLoS One. 2011]</span><div class="brieflinkpop offscreen_noflow"><span xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="invert">Review</span> Applying the risk of bias tool in a systematic review of combination long-acting beta-agonists and inhaled corticosteroids for persistent asthma.<div class="brieflinkpopdesc"><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="author">Hartling L, Bond K, Vandermeer B, Seida J, Dryden DM, Rowe BH. </em><em xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" class="cit">PLoS One. 2011 Feb 24; 6(2):e17242. Epub 2011 Feb 24.</em></div></div></li></ul><a class="seemore" href="/sites/entrez?db=pubmed&cmd=link&linkname=pubmed_pubmed_reviews&uid=22536612" ref="ordinalpos=1&log$=relatedreviews_seeall&logdbfrom=pubmed">See reviews...</a><a class="seemore" href="/sites/entrez?db=pubmed&cmd=link&linkname=pubmed_pubmed&uid=22536612" ref="ordinalpos=1&log$=relatedarticles_seeall&logdbfrom=pubmed">See all...</a></div></div><div class="portlet"><div class="portlet_head"><div class="portlet_title"><h3><span>Recent Activity</span></h3></div><a name="Shutter" sid="1" href="#" class="portlet_shutter" title="Show/hide content" remembercollapsed="true" pgsec_name="recent_activity" id="Shutter"></a></div><div class="portlet_content"><div xmlns:np="http://ncbi.gov/portal/XSLT/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="HTDisplay" class=""><div class="action"><a href="javascript:historyDisplayState('ClearHT')">Clear</a><a href="javascript:historyDisplayState('HTOff')" class="HTOn">Turn Off</a><a href="javascript:historyDisplayState('HTOn')" class="HTOff">Turn On</a></div><ul id="activity"><li class="ra_rcd ralinkpopper two_line"><a class="htb ralinkpopperctrl" ref="log$=activity&linkpos=1" href="/portal/utils/pageresolver.fcgi?recordid=67d6b60ccde49f3df70f05ec">Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments</a><div class="ralinkpop offscreen_noflow">Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments<div class="brieflinkpopdesc"></div></div><div class="tertiary"></div></li><li class="ra_rcd ralinkpopper two_line"><a class="htb ralinkpopperctrl" ref="log$=activity&linkpos=2" href="/portal/utils/pageresolver.fcgi?recordid=67d6b60b67c23b31e0e46d85">Usefulness of Economic Evaluation Data in Systematic Reviews of Evidence</a><div class="ralinkpop offscreen_noflow">Usefulness of Economic Evaluation Data in Systematic Reviews of Evidence<div class="brieflinkpopdesc"></div></div><div class="tertiary"></div></li><li class="ra_rcd ralinkpopper two_line"><a class="htb ralinkpopperctrl" ref="log$=activity&linkpos=3" href="/portal/utils/pageresolver.fcgi?recordid=67d6b60acde49f3df70efc1d">Use of Mixed Treatment Comparisons in Systematic Reviews</a><div class="ralinkpop offscreen_noflow">Use of Mixed Treatment Comparisons in Systematic Reviews<div class="brieflinkpopdesc"></div></div><div class="tertiary"></div></li><li class="ra_rcd ralinkpopper two_line"><a class="htb ralinkpopperctrl" ref="log$=activity&linkpos=4" href="/portal/utils/pageresolver.fcgi?recordid=67d6b60967c23b31e0e467bf">Systematizing the Use of Value of Information Analysis in Prioritizing Systemati...</a><div class="ralinkpop offscreen_noflow">Systematizing the Use of Value of Information Analysis in Prioritizing Systematic Reviews<div class="brieflinkpopdesc"></div></div><div class="tertiary"></div></li><li class="ra_rcd ralinkpopper two_line"><a class="htb ralinkpopperctrl" ref="log$=activity&linkpos=5" href="/portal/utils/pageresolver.fcgi?recordid=67d6b60767c23b31e0e46771">Selective Outcome Reporting as a Source of Bias in Reviews of Comparative Effect...</a><div class="ralinkpop offscreen_noflow">Selective Outcome Reporting as a Source of Bias in Reviews of Comparative Effectiveness<div class="brieflinkpopdesc"></div></div><div class="tertiary"></div></li></ul><p class="HTOn">Your browsing activity is empty.</p><p class="HTOff">Activity recording is turned off.</p><p id="turnOn" class="HTOff"><a href="javascript:historyDisplayState('HTOn')">Turn recording back on</a></p><a class="seemore" href="/sites/myncbi/recentactivity">See more...</a></div></div></div>
|
||
|
||
<!-- Custom content below discovery portlets -->
|
||
<div class="col7">
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Custom content after all -->
|
||
<div class="col8">
|
||
|
||
</div>
|
||
<div class="col9">
|
||
|
||
</div>
|
||
|
||
<script type="text/javascript" src="/corehtml/pmc/js/jquery.scrollTo-1.4.2.js"></script>
|
||
<script type="text/javascript">
|
||
(function($){
|
||
$('.skiplink').each(function(i, item){
|
||
var href = $($(item).attr('href'));
|
||
href.attr('tabindex', '-1').addClass('skiptarget'); // ensure the target can receive focus
|
||
$(item).on('click', function(event){
|
||
event.preventDefault();
|
||
$.scrollTo(href, 0, {
|
||
onAfter: function(){
|
||
href.focus();
|
||
}
|
||
});
|
||
});
|
||
});
|
||
})(jQuery);
|
||
</script>
|
||
</div>
|
||
<div class="bottom">
|
||
<script type="text/javascript">
|
||
var PBooksSearchTermData = {
|
||
highlighter: "bold",
|
||
dateTime: "03/16/2025 05:16:39",
|
||
terms: [
|
||
'degeneration', 'disease', 'hepatolenticular', 'hepatolenticular degeneration', 'hepatolenticular degeneration', 'practice guideline', 'wilson', 'wilson disease'
|
||
]
|
||
};
|
||
</script>
|
||
<div id="NCBIFooter_dynamic">
|
||
<!--<component id="Breadcrumbs" label="breadcrumbs"/>
|
||
<component id="Breadcrumbs" label="helpdesk"/>-->
|
||
|
||
</div>
|
||
|
||
<div class="footer" id="footer">
|
||
<section class="icon-section">
|
||
<div id="icon-section-header" class="icon-section_header">Follow NCBI</div>
|
||
<div class="grid-container container">
|
||
<div class="icon-section_container">
|
||
<a class="footer-icon" id="footer_twitter" href="https://twitter.com/ncbi" aria-label="Twitter"><svg xmlns="http://www.w3.org/2000/svg" data-name="Layer 1" viewBox="0 0 300 300">
|
||
<defs>
|
||
<style>
|
||
.cls-11 {
|
||
fill: #737373;
|
||
}
|
||
</style>
|
||
</defs>
|
||
<title>Twitter</title>
|
||
<path class="cls-11" d="M250.11,105.48c-7,3.14-13,3.25-19.27.14,8.12-4.86,8.49-8.27,11.43-17.46a78.8,78.8,0,0,1-25,9.55,39.35,39.35,0,0,0-67,35.85,111.6,111.6,0,0,1-81-41.08A39.37,39.37,0,0,0,81.47,145a39.08,39.08,0,0,1-17.8-4.92c0,.17,0,.33,0,.5a39.32,39.32,0,0,0,31.53,38.54,39.26,39.26,0,0,1-17.75.68,39.37,39.37,0,0,0,36.72,27.3A79.07,79.07,0,0,1,56,223.34,111.31,111.31,0,0,0,116.22,241c72.3,0,111.83-59.9,111.83-111.84,0-1.71,0-3.4-.1-5.09C235.62,118.54,244.84,113.37,250.11,105.48Z">
|
||
</path>
|
||
</svg></a>
|
||
<a class="footer-icon" id="footer_facebook" href="https://www.facebook.com/ncbi.nlm" aria-label="Facebook"><svg xmlns="http://www.w3.org/2000/svg" data-name="Layer 1" viewBox="0 0 300 300">
|
||
<title>Facebook</title>
|
||
<path class="cls-11" d="M210.5,115.12H171.74V97.82c0-8.14,5.39-10,9.19-10h27.14V52l-39.32-.12c-35.66,0-42.42,26.68-42.42,43.77v19.48H99.09v36.32h27.24v109h45.41v-109h35Z">
|
||
</path>
|
||
</svg></a>
|
||
<a class="footer-icon" id="footer_linkedin" href="https://www.linkedin.com/company/ncbinlm" aria-label="LinkedIn"><svg xmlns="http://www.w3.org/2000/svg" data-name="Layer 1" viewBox="0 0 300 300">
|
||
<title>LinkedIn</title>
|
||
<path class="cls-11" d="M101.64,243.37H57.79v-114h43.85Zm-22-131.54h-.26c-13.25,0-21.82-10.36-21.82-21.76,0-11.65,8.84-21.15,22.33-21.15S101.7,78.72,102,90.38C102,101.77,93.4,111.83,79.63,111.83Zm100.93,52.61A17.54,17.54,0,0,0,163,182v61.39H119.18s.51-105.23,0-114H163v13a54.33,54.33,0,0,1,34.54-12.66c26,0,44.39,18.8,44.39,55.29v58.35H198.1V182A17.54,17.54,0,0,0,180.56,164.44Z">
|
||
</path>
|
||
</svg></a>
|
||
<a class="footer-icon" id="footer_github" href="https://github.com/ncbi" aria-label="GitHub"><svg xmlns="http://www.w3.org/2000/svg" data-name="Layer 1" viewBox="0 0 300 300">
|
||
<defs>
|
||
<style>
|
||
.cls-11,
|
||
.cls-12 {
|
||
fill: #737373;
|
||
}
|
||
|
||
.cls-11 {
|
||
fill-rule: evenodd;
|
||
}
|
||
</style>
|
||
</defs>
|
||
<title>GitHub</title>
|
||
<path class="cls-11" d="M151.36,47.28a105.76,105.76,0,0,0-33.43,206.1c5.28,1,7.22-2.3,7.22-5.09,0-2.52-.09-10.85-.14-19.69-29.42,6.4-35.63-12.48-35.63-12.48-4.81-12.22-11.74-15.47-11.74-15.47-9.59-6.56.73-6.43.73-6.43,10.61.75,16.21,10.9,16.21,10.9,9.43,16.17,24.73,11.49,30.77,8.79,1-6.83,3.69-11.5,6.71-14.14C108.57,197.1,83.88,188,83.88,147.51a40.92,40.92,0,0,1,10.9-28.39c-1.1-2.66-4.72-13.42,1-28,0,0,8.88-2.84,29.09,10.84a100.26,100.26,0,0,1,53,0C198,88.3,206.9,91.14,206.9,91.14c5.76,14.56,2.14,25.32,1,28a40.87,40.87,0,0,1,10.89,28.39c0,40.62-24.74,49.56-48.29,52.18,3.79,3.28,7.17,9.71,7.17,19.58,0,14.15-.12,25.54-.12,29,0,2.82,1.9,6.11,7.26,5.07A105.76,105.76,0,0,0,151.36,47.28Z">
|
||
</path>
|
||
<path class="cls-12" d="M85.66,199.12c-.23.52-1.06.68-1.81.32s-1.2-1.06-.95-1.59,1.06-.69,1.82-.33,1.21,1.07.94,1.6Zm-1.3-1">
|
||
</path>
|
||
<path class="cls-12" d="M90,203.89c-.51.47-1.49.25-2.16-.49a1.61,1.61,0,0,1-.31-2.19c.52-.47,1.47-.25,2.17.49s.82,1.72.3,2.19Zm-1-1.08">
|
||
</path>
|
||
<path class="cls-12" d="M94.12,210c-.65.46-1.71,0-2.37-.91s-.64-2.07,0-2.52,1.7,0,2.36.89.65,2.08,0,2.54Zm0,0"></path>
|
||
<path class="cls-12" d="M99.83,215.87c-.58.64-1.82.47-2.72-.41s-1.18-2.06-.6-2.7,1.83-.46,2.74.41,1.2,2.07.58,2.7Zm0,0">
|
||
</path>
|
||
<path class="cls-12" d="M107.71,219.29c-.26.82-1.45,1.2-2.64.85s-2-1.34-1.74-2.17,1.44-1.23,2.65-.85,2,1.32,1.73,2.17Zm0,0">
|
||
</path>
|
||
<path class="cls-12" d="M116.36,219.92c0,.87-1,1.59-2.24,1.61s-2.29-.68-2.3-1.54,1-1.59,2.26-1.61,2.28.67,2.28,1.54Zm0,0">
|
||
</path>
|
||
<path class="cls-12" d="M124.42,218.55c.15.85-.73,1.72-2,1.95s-2.37-.3-2.52-1.14.73-1.75,2-2,2.37.29,2.53,1.16Zm0,0"></path>
|
||
</svg></a>
|
||
<a class="footer-icon" id="footer_blog" href="https://ncbiinsights.ncbi.nlm.nih.gov/" aria-label="Blog">
|
||
<svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" data-name="Layer 1" viewBox="0 0 40 40">
|
||
<defs><style>.cls-1{fill:#737373;}</style></defs>
|
||
<title>NCBI Insights Blog</title>
|
||
<path class="cls-1" d="M14,30a4,4,0,1,1-4-4,4,4,0,0,1,4,4Zm11,3A19,19,0,0,0,7.05,15a1,1,0,0,0-1,1v3a1,1,0,0,0,.93,1A14,14,0,0,1,20,33.07,1,1,0,0,0,21,34h3a1,1,0,0,0,1-1Zm9,0A28,28,0,0,0,7,6,1,1,0,0,0,6,7v3a1,1,0,0,0,1,1A23,23,0,0,1,29,33a1,1,0,0,0,1,1h3A1,1,0,0,0,34,33Z"></path>
|
||
</svg>
|
||
</a>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="container-fluid bg-primary">
|
||
<div class="container pt-5">
|
||
<div class="row mt-3">
|
||
<div class="col-lg-3 col-12">
|
||
<p><a class="text-white" href="https://www.nlm.nih.gov/socialmedia/index.html">Connect with NLM</a></p>
|
||
<ul class="list-inline social_media">
|
||
<li class="list-inline-item"><a href="https://twitter.com/NLM_NIH" aria-label="Twitter" target="_blank" rel="noopener noreferrer"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" x="0px" y="0px" viewBox="0 0 249 249" style="enable-background:new 0 0 249 249;" xml:space="preserve">
|
||
<style type="text/css">
|
||
.st20 {
|
||
fill: #FFFFFF;
|
||
}
|
||
|
||
.st30 {
|
||
fill: none;
|
||
stroke: #FFFFFF;
|
||
stroke-width: 8;
|
||
stroke-miterlimit: 10;
|
||
}
|
||
</style>
|
||
<title>Twitter</title>
|
||
<g>
|
||
<g>
|
||
<g>
|
||
<path class="st20" d="M192.9,88.1c-5,2.2-9.2,2.3-13.6,0.1c5.7-3.4,6-5.8,8.1-12.3c-5.4,3.2-11.4,5.5-17.6,6.7 c-10.5-11.2-28.1-11.7-39.2-1.2c-7.2,6.8-10.2,16.9-8,26.5c-22.3-1.1-43.1-11.7-57.2-29C58,91.6,61.8,107.9,74,116 c-4.4-0.1-8.7-1.3-12.6-3.4c0,0.1,0,0.2,0,0.4c0,13.2,9.3,24.6,22.3,27.2c-4.1,1.1-8.4,1.3-12.5,0.5c3.6,11.3,14,19,25.9,19.3 c-11.6,9.1-26.4,13.2-41.1,11.5c12.7,8.1,27.4,12.5,42.5,12.5c51,0,78.9-42.2,78.9-78.9c0-1.2,0-2.4-0.1-3.6 C182.7,97.4,189.2,93.7,192.9,88.1z"></path>
|
||
</g>
|
||
</g>
|
||
<circle class="st30" cx="124.4" cy="128.8" r="108.2"></circle>
|
||
</g>
|
||
</svg></a></li>
|
||
<li class="list-inline-item"><a href="https://www.facebook.com/nationallibraryofmedicine" aria-label="Facebook" rel="noopener noreferrer" target="_blank">
|
||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" x="0px" y="0px" viewBox="0 0 249 249" style="enable-background:new 0 0 249 249;" xml:space="preserve">
|
||
<style type="text/css">
|
||
.st10 {
|
||
fill: #FFFFFF;
|
||
}
|
||
|
||
.st110 {
|
||
fill: none;
|
||
stroke: #FFFFFF;
|
||
stroke-width: 8;
|
||
stroke-miterlimit: 10;
|
||
}
|
||
</style>
|
||
<title>Facebook</title>
|
||
<g>
|
||
<g>
|
||
<path class="st10" d="M159,99.1h-24V88.4c0-5,3.3-6.2,5.7-6.2h16.8V60l-24.4-0.1c-22.1,0-26.2,16.5-26.2,27.1v12.1H90v22.5h16.9 v67.5H135v-67.5h21.7L159,99.1z"></path>
|
||
</g>
|
||
</g>
|
||
<circle class="st110" cx="123.6" cy="123.2" r="108.2"></circle>
|
||
</svg>
|
||
</a></li>
|
||
<li class="list-inline-item"><a href="https://www.youtube.com/user/NLMNIH" aria-label="Youtube" target="_blank" rel="noopener noreferrer"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" x="0px" y="0px" viewBox="0 0 249 249" style="enable-background:new 0 0 249 249;" xml:space="preserve">
|
||
<title>Youtube</title>
|
||
<style type="text/css">
|
||
.st4 {
|
||
fill: none;
|
||
stroke: #FFFFFF;
|
||
stroke-width: 8;
|
||
stroke-miterlimit: 10;
|
||
}
|
||
|
||
.st5 {
|
||
fill: #FFFFFF;
|
||
}
|
||
</style>
|
||
<circle class="st4" cx="124.2" cy="123.4" r="108.2"></circle>
|
||
<g transform="translate(0,-952.36218)">
|
||
<path class="st5" d="M88.4,1037.4c-10.4,0-18.7,8.3-18.7,18.7v40.1c0,10.4,8.3,18.7,18.7,18.7h72.1c10.4,0,18.7-8.3,18.7-18.7 v-40.1c0-10.4-8.3-18.7-18.7-18.7H88.4z M115.2,1058.8l29.4,17.4l-29.4,17.4V1058.8z"></path>
|
||
</g>
|
||
</svg></a></li>
|
||
</ul>
|
||
</div>
|
||
<div class="col-lg-3 col-12">
|
||
<p class="address_footer text-white">National Library of Medicine<br />
|
||
<a href="https://www.google.com/maps/place/8600+Rockville+Pike,+Bethesda,+MD+20894/@38.9959508,-77.101021,17z/data=!3m1!4b1!4m5!3m4!1s0x89b7c95e25765ddb:0x19156f88b27635b8!8m2!3d38.9959508!4d-77.0988323" class="text-white" target="_blank" rel="noopener noreferrer">8600 Rockville Pike<br />
|
||
Bethesda, MD 20894</a></p>
|
||
</div>
|
||
<div class="col-lg-3 col-12 centered-lg">
|
||
<p><a href="https://www.nlm.nih.gov/web_policies.html" class="text-white">Web Policies</a><br />
|
||
<a href="https://www.nih.gov/institutes-nih/nih-office-director/office-communications-public-liaison/freedom-information-act-office" class="text-white">FOIA</a><br />
|
||
<a href="https://www.hhs.gov/vulnerability-disclosure-policy/index.html" class="text-white" id="vdp">HHS Vulnerability Disclosure</a></p>
|
||
</div>
|
||
<div class="col-lg-3 col-12 centered-lg">
|
||
<p><a class="supportLink text-white" href="https://support.nlm.nih.gov/">Help</a><br />
|
||
<a href="https://www.nlm.nih.gov/accessibility.html" class="text-white">Accessibility</a><br />
|
||
<a href="https://www.nlm.nih.gov/careers/careers.html" class="text-white">Careers</a></p>
|
||
</div>
|
||
</div>
|
||
<div class="row">
|
||
<div class="col-lg-12 centered-lg">
|
||
<nav class="bottom-links">
|
||
<ul class="mt-3">
|
||
<li>
|
||
<a class="text-white" href="//www.nlm.nih.gov/">NLM</a>
|
||
</li>
|
||
<li>
|
||
<a class="text-white" href="https://www.nih.gov/">NIH</a>
|
||
</li>
|
||
<li>
|
||
<a class="text-white" href="https://www.hhs.gov/">HHS</a>
|
||
</li>
|
||
<li>
|
||
<a class="text-white" href="https://www.usa.gov/">USA.gov</a>
|
||
</li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<script type="text/javascript" src="/portal/portal3rc.fcgi/rlib/js/InstrumentOmnitureBaseJS/InstrumentNCBIConfigJS/InstrumentNCBIBaseJS/InstrumentPageStarterJS.js?v=1"> </script>
|
||
<script type="text/javascript" src="/portal/portal3rc.fcgi/static/js/hfjs2.js"> </script>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<!--/.page-->
|
||
</div>
|
||
<!--/.wrap-->
|
||
</div><!-- /.twelve_col -->
|
||
</div>
|
||
<!-- /.grid -->
|
||
|
||
<span class="PAFAppResources"></span>
|
||
|
||
<!-- BESelector tab -->
|
||
|
||
|
||
|
||
<noscript><img alt="statistics" src="/stat?jsdisabled=true&ncbi_db=books&ncbi_pdid=book-toc&ncbi_acc=NBK92293&ncbi_domain=methresvalid&ncbi_report=record&ncbi_type=fulltext&ncbi_objectid=&ncbi_pcid=/NBK92293/&ncbi_pagename=Validity and Inter-Rater Reliability Testing of Quality Assessment Instruments - NCBI Bookshelf&ncbi_bookparttype=toc&ncbi_app=bookshelf" /></noscript>
|
||
|
||
|
||
<!-- usually for JS scripts at page bottom -->
|
||
<!--<component id="PageFixtures" label="styles"></component>-->
|
||
|
||
|
||
<!-- CE8B5AF87C7FFCB1_0191SID /projects/books/PBooks@9.11 portal106 v4.1.r689238 Tue, Oct 22 2024 16:10:51 -->
|
||
<span id="portal-csrf-token" style="display:none" data-token="CE8B5AF87C7FFCB1_0191SID"></span>
|
||
|
||
<script type="text/javascript" src="//static.pubmed.gov/portal/portal3rc.fcgi/4216699/js/3879255/4121861/3501987/4008961/3893018/3821238/4062932/4209313/4212053/4076480/3921943/3400083/3426610.js" snapshot="books"></script></body>
|
||
</html> |