Difference between revisions of "REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions (Summer 2007)"

From LearnLab
Jump to: navigation, search
Line 1: Line 1:
−
 
+
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 +
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr">
 +
<head>
 +
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 +
<meta name="keywords" content=" REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions (Summer 2007) " />
 +
<link rel="shortcut icon" href="/favicon.ico" />
 +
<title> REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions (Summer 2007)  - Pslc</title>
 +
<style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "/research/wiki/skins/monobook/main.css?7"; /*]]>*/</style>
 +
<link rel="stylesheet" type="text/css" media="print" href="/research/wiki/skins/common/commonPrint.css" />
 +
<!--[if lt IE 5.5000]><style type="text/css">@import "/research/wiki/skins/monobook/IE50Fixes.css";</style><![endif]-->
 +
<!--[if IE 5.5000]><style type="text/css">@import "/research/wiki/skins/monobook/IE55Fixes.css";</style><![endif]-->
 +
<!--[if IE 6]><style type="text/css">@import "/research/wiki/skins/monobook/IE60Fixes.css";</style><![endif]-->
 +
<!--[if IE 7]><style type="text/css">@import "/research/wiki/skins/monobook/IE70Fixes.css?1";</style><![endif]-->
 +
<!--[if lt IE 7]><script type="text/javascript" src="/research/wiki/skins/common/IEFixes.js"></script>
 +
<meta http-equiv="imagetoolbar" content="no" /><![endif]-->
 +
<script type="text/javascript">var skin = 'monobook';var stylepath = '/research/wiki/skins';</script>
 +
<script type="text/javascript" src="/research/wiki/skins/common/wikibits.js"><!-- wikibits js --></script>
 +
<script type="text/javascript" src="/research/wiki/index.php?title=-&amp;action=raw&amp;smaxage=0&amp;gen=js"><!-- site js --></script>
 +
<style type="text/css">/*<![CDATA[*/
 +
@import "/research/wiki/index.php?title=MediaWiki:Common.css&action=raw&ctype=text/css&smaxage=18000";
 +
@import "/research/wiki/index.php?title=MediaWiki:Monobook.css&action=raw&ctype=text/css&smaxage=18000";
 +
@import "/research/wiki/index.php?title=-&action=raw&gen=css&maxage=18000";
 +
/*]]>*/</style>
 +
<!-- Head Scripts -->
 +
</head>
 +
<body  class="ns-0 ltr">
 +
<div id="globalWrapper">
 +
<div id="column-content">
 +
<div id="content">
 +
<a name="top" id="top"></a>
 +
<h1 class="firstHeading"> REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions (Summer 2007) </h1>
 +
<div id="bodyContent">
 +
<h3 id="siteSub">From Pslc</h3>
 +
<div id="contentSub"></div>
 +
<div id="jump-to-nav">Jump to: <a href="#column-one">navigation</a>, <a href="#searchInput">search</a></div> <!-- start content -->
 +
<table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div>
 +
<ul>
 +
<li class='toclevel-1'><a href="# Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions"><span class="tocnumber">1</span> <span class="toctext"> REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions </span></a>
 +
<ul>
 +
<li class='toclevel-2'><a href="#Logistical_Information"><span class="tocnumber">1.1</span> <span class="toctext">Logistical Information</span></a></li>
 +
<li class='toclevel-2'><a href="#Abstract"><span class="tocnumber">1.2</span> <span class="toctext">Abstract</span></a></li>
 +
<li class='toclevel-2'><a href="#Glossary"><span class="tocnumber">1.3</span> <span class="toctext">Glossary</span></a></li>
 +
<li class='toclevel-2'><a href="#Research_question"><span class="tocnumber">1.4</span> <span class="toctext">Research question</span></a></li>
 +
<li class='toclevel-2'><a href="#Dependent_variables"><span class="tocnumber">1.5</span> <span class="toctext">Dependent variables</span></a></li>
 +
<li class='toclevel-2'><a href="#Independent_variables"><span class="tocnumber">1.6</span> <span class="toctext">Independent variables</span></a></li>
 +
<li class='toclevel-2'><a href="#Hypotheses"><span class="tocnumber">1.7</span> <span class="toctext">Hypotheses</span></a></li>
 +
<li class='toclevel-2'><a href="#Explanation"><span class="tocnumber">1.8</span> <span class="toctext">Explanation</span></a></li>
 +
<li class='toclevel-2'><a href="#Descendents"><span class="tocnumber">1.9</span> <span class="toctext">Descendents</span></a></li>
 +
<li class='toclevel-2'><a href="#Notes"><span class="tocnumber">1.10</span> <span class="toctext">Notes</span></a></li>
 +
<li class='toclevel-2'><a href="#Annotated_bibliography"><span class="tocnumber">1.11</span> <span class="toctext">Annotated bibliography</span></a></li>
 +
</ul>
 +
</li>
 +
</ul>
 +
</td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions%28Summer_2007%29&amp;action=edit&amp;section=1" title="Edit section: REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions ">edit</a>]</div><a name=" REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions "></a><h2> REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions </h2>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&amp;action=edit&amp;section=2" title="Edit section: Logistical Information">edit</a>]</div><a name="Logistical_Information"></a><h3> Logistical Information </h3>
 +
<table border="1">
 +
<caption>
 +
</caption>
 +
<tr>
 +
<td> <b>Contributors</b> </td><td> Christine M. Feeney and Michael Heilman
 +
</td></tr>
 +
<tr>
 +
<td> <b>Study Start Date</b> </td><td> June, 2007
 +
</td></tr>
 +
<tr>
 +
<td> <b>Study End Date</b> </td><td> July, 2007
 +
</td></tr>
 +
<tr>
 +
<td> <b>Learnlab Courses</b> </td><td>none
 +
</td></tr>
 +
<tr>
 +
<td> <b>Number of Students</b> </td><td>  30
 +
</td></tr>
 +
<tr>
 +
<td> <b>Total Participant Hours (est.)</b> </td><td>  300
 +
</td></tr>
 +
<tr>
 +
<td> <b>Data in Datashop</b> </td><td> no
 +
</td></tr></table>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&amp;action=edit&amp;section=3" title="Edit section: Abstract">edit</a>]</div><a name="Abstract"></a><h3> Abstract </h3>
 +
<p> Previous psychological research has identified two types of comprehension: shallow, in which people can reproduce information, and deep, in which people can comprehend the meaning of information.  In addition, researchers have found that these two types of comprehension are separate and disparate.  The PSLC currently employs an English as a Second Language vocabulary tutor, REAP, that uses shallow, automatically generated, comprehension questions to check that students are actively reading, rather than just skimming, practice reading passages.  The purpose of the current study was to examine performance on the REAP-type questions with manually authored, deeper, reading comprehension questions.  Participants were thirty undergraduate students (male = 11) participating in summer research programs at Carnegie Mellon University.  The researcher predicted a positive correlation between the two testing types, which was supported by the data (r = .366, p < 0.0005, one-tailed t-test). 
 +
</p>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&amp;action=edit&amp;section=4" title="Edit section: Glossary">edit</a>]</div><a name="Glossary"></a><h3> Glossary </h3>
 +
<p><i>Shallow Comprehension:</i> Processing is directed towards reproducing the learning material without necessarily understanding it.
 +
</p>
 +
<p>
 +
<i> Deep Comprehension:</i> Processing is directed towards comprehending the intended meaning of the learning material.
 +
</p>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%%28Summer_2007%29&amp;action=edit&amp;section=5" title="Edit section: Research question">edit</a>]</div><a name="Research_question"></a><h3> Research question </h3>
 +
<p> Does performance on the shallow, automatically-generated questions correlate with performance on more sophisticated, deep reading comprehension questions?
 +
</p>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&amp;action=edit&amp;section=6" title="Edit section: Dependent variables">edit</a>]</div><a name="Dependent_variables"></a><h3> Dependent variables </h3>
 +
<p> Percentage of Shallow Comprehension Questions Answered Correctly.
 +
</p><p> Percentage of Deep Comprehension Questions Answered Correctly.
 +
</p>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&amp;action=edit&amp;section=7" title="Edit section: Independent variables">edit</a>]</div><a name="Independent_variables"></a><h3> Independent variable </h3>
 +
<p>Five reading passages, each of which was followed by four shallow reading comprehension questions and three to five deep reading comprehension questions.
 +
</p>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&amp;action=edit&amp;section=8" title="Edit section: Hypotheses">edit</a>]</div><a name="Hypotheses"></a><h3> Hypotheses </h3>
 +
<p> A significant positive correlation between shallow and deep reading comprehension questions will occur in this study.
 +
</p>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&amp;action=edit&amp;section=9" title="Edit section: Explanation">edit</a>]</div><a name="Explanation"></a><h3> Explanation </h3>
 +
<p>The study found no significant differences between the two conditions (2 or 4 words/text).  This null result may be due to statistical power and mistakes in the study design.  The tutor had difficulty providing high quality readings with four vocabulary words.  Thus, the tutor often had to give texts with 3 or even 2 words to students in the 4-word condition.  Additionally, the optimal number of words per text (the density of practice) may be higher than 4 words.  There may be a threshold at which the density of practice becomes too high.  This threshold may actually be at 5 or even 10 words given the average length of the texts used by the REAP tutor.  Some research suggests that up to 5% of the words in a text can be unknown before comprehension is impeded (Laufer, 1992).  Although the task for that study was different (reading comprehension rather than vocabulary acquisition), it suggests that the optimal level of stretch may be such that as many as 20 words may be unknown in a text with 400 word types (which is approximately the mean number of word types in REAP texts).  Other research, however, suggests higher thresholds up to 98% (Nation, 2001).
 +
</p><p>Although the results for stretch from this study were inconclusive, a great deal of interesting data were gathered.  Some interesting results have been found, and further analysis is possible.
 +
</p><p>Many potential variables exist that could lead to successful learning of new vocabulary: general ESL ability as measured by the MTELP, time on task, number of words looked up, pre-test scores on the academic word list. A step-wise linear regression showed that the most significant predictor of acquisition of new vocabulary was the size of the student's vocabulary scores on the academic word list prior to instruction (r= 0.77, p &lt; .0001). However, the effect of this prior knowledge was diminished by the number of texts the students read r= -0.17). Together, in the stepwise regression, these two variable accounted for 67% of the variance (r2= .67).
 +
</p><p>This finding corroborates findings of Stanovich (1986) and James (1996) who refer to the phenomenon as the "Matthew effect" (after a Bible passage), whereby the rich get richer and the poor get poorer. In this case, it is not 'general richness' expressed in terms of higher general ESL proficiency, but specifically knowledge of words on the academic word list. In other words, the more knowledge components in the domain of acquisition a learner brings to the task, the greater the potential for future learning.
 +
</p><p>It was also found that students frequently access dictionary definitions for non-target words that are well below their expected reading level (e.g., "hitting").  This may lead to revisions of curricula for ESL vocabulary learning.
 +
</p>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&amp;action=edit&amp;section=10" title="Edit section: Descendents">edit</a>]</div><a name="Descendents"></a><h3> Descendents </h3>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&amp;action=edit&amp;section=11" title="Edit section: Notes">edit</a>]</div><a name="Notes"></a><h3> Notes </h3>
 +
<p>Target vocabulary lists were determined by a lengthy pre-test using multiple choice cloze questions.  Each question takes 30-60 seconds on average, depending on the student.  In later studies, self-assessment pre-tests were preferred due to time constraints and initial impressions of the students.  Lengthy pre-tests for determining target word lists leave a poor initial impression of the tutor.
 +
</p>
 +
<div class="editsection" style="float:right;margin-left:5px;">[<a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&amp;action=edit&amp;section=12" title="Edit section: Annotated bibliography">edit</a>]</div><a name="Annotated_bibliography"></a><h3> Annotated bibliography </h3>
 +
<p>Laufer, B. (1992). How much lexis is necessary for reading comprehension? In Vocabulary and Applied Linguistics, Pierre J. L. Arnaud and Henri Béjoint (eds.), 126–132. London: Macmillan.
 +
</p>
 +
<!-- Saved in parser cache with key theory_wiki:pcache:idhash:1510-0!1!0!0!!en!2 and timestamp 20070725163805 -->
 +
<div class="printfooter">
 +
Retrieved from "<a href="http://www.learnlab.org/research/wiki/index.php/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">http://www.learnlab.org/research/wiki/index.php/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29</a>"</div>
 +
<!-- end content -->
 +
<div class="visualClear"></div>
 +
</div>
 +
</div>
 +
</div>
 +
<div id="column-one">
 +
<div id="p-cactions" class="portlet">
 +
<h5>Views</h5>
 +
<ul>
 +
<li id="ca-nstab-main" class="selected"><a href="/research/wiki/index.php/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">Article</a></li>
 +
<li id="ca-talk" class="new"><a href="/research/wiki/index.php?title=Talk:REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&amp;action=edit">Discussion</a></li>
 +
<li id="ca-edit"><a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&amp;action=edit">Edit</a></li>
 +
<li id="ca-history"><a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&amp;action=history">History</a></li>
 +
<li id="ca-move"><a href="/research/wiki/index.php/Special:Movepage/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">Move</a></li>
 +
<li id="ca-watch"><a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&amp;action=watch">Watch</a></li>
 +
</ul>
 +
</div>
 +
<div class="portlet" id="p-personal">
 +
<h5>Personal tools</h5>
 +
<div class="pBody">
 +
<ul>
 +
<li id="pt-userpage"><a href="/research/wiki/index.php/User:Feeney" class="new">Feeney</a></li>
 +
<li id="pt-mytalk"><a href="/research/wiki/index.php/User_talk:Feeney" class="new">My talk</a></li>
 +
<li id="pt-preferences"><a href="/research/wiki/index.php/Special:Preferences">Preferences</a></li>
 +
<li id="pt-watchlist"><a href="/research/wiki/index.php/Special:Watchlist">My watchlist</a></li>
 +
<li id="pt-mycontris"><a href="/research/wiki/index.php/Special:Contributions/Feeney">My contributions</a></li>
 +
<li id="pt-logout"><a href="/research/wiki/index.php?title=Special:Userlogout&amp;returnto=REAP_Study_on_Vocabulary_Stretch_(Spring_2006)">Log out</a></li>
 +
</ul>
 +
</div>
 +
</div>
 +
<div class="portlet" id="p-logo">
 +
<a style="background-image: url(/research/wiki/skins/pslc_logo.png);" href="/research/wiki/index.php/Main_Page" title="Main Page"></a>
 +
</div>
 +
<script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script>
 +
<div class='portlet' id='p-navigation'>
 +
<h5>Navigation</h5>
 +
<div class='pBody'>
 +
<ul>
 +
<li id="n-mainpage"><a href="/research/wiki/index.php/Main_Page">Main Page</a></li>
 +
<li id="n-Root-Node"><a href="/research/wiki/index.php/Root_node">Root Node</a></li>
 +
<li id="n-Glossary"><a href="/research/wiki/index.php/Category:Glossary">Glossary</a></li>
 +
<li id="n-recentchanges"><a href="/research/wiki/index.php/Special:Recentchanges">Recent changes</a></li>
 +
<li id="n-randompage"><a href="/research/wiki/index.php/Special:Random">Random page</a></li>
 +
<li id="n-help"><a href="/research/wiki/index.php/Help:Contents">Help</a></li>
 +
</ul>
 +
</div>
 +
</div>
 +
<div class='portlet' id='p-clusters'>
 +
<h5>clusters</h5>
 +
<div class='pBody'>
 +
<ul>
 +
<li id="n-Coordinative-Learning"><a href="/research/wiki/index.php/Coordinative_Learning">Coordinative Learning</a></li>
 +
<li id="n-Interactive-Communication"><a href="/research/wiki/index.php/Interactive_Communication">Interactive Communication</a></li>
 +
<li id="n-Refinement-and-Fluency"><a href="/research/wiki/index.php/Refinement_and_Fluency">Refinement and Fluency</a></li>
 +
</ul>
 +
</div>
 +
</div>
 +
<div class='portlet' id='p-courses'>
 +
<h5>courses</h5>
 +
<div class='pBody'>
 +
<ul>
 +
<li id="n-Algebra"><a href="/research/wiki/index.php/Algebra">Algebra</a></li>
 +
<li id="n-Geometry"><a href="/research/wiki/index.php/Geometry">Geometry</a></li>
 +
<li id="n-Chinese"><a href="/research/wiki/index.php/Chinese">Chinese</a></li>
 +
<li id="n-French"><a href="/research/wiki/index.php/French">French</a></li>
 +
<li id="n-English"><a href="/research/wiki/index.php/English">English</a></li>
 +
<li id="n-Chemistry"><a href="/research/wiki/index.php/Chemistry">Chemistry</a></li>
 +
<li id="n-Physics"><a href="/research/wiki/index.php/Physics">Physics</a></li>
 +
</ul>
 +
</div>
 +
</div>
 +
<div class='portlet' id='p-datashop'>
 +
<h5>datashop</h5>
 +
<div class='pBody'>
 +
<ul>
 +
<li id="n-DataShop-Pipeline"><a href="/research/wiki/index.php/DataShop_Pipeline">DataShop Pipeline</a></li>
 +
</ul>
 +
</div>
 +
</div>
 +
<div id="p-search" class="portlet">
 +
<h5><label for="searchInput">Search</label></h5>
 +
<div id="searchBody" class="pBody">
 +
<form action="/research/wiki/index.php/Special:Search" id="searchform"><div>
 +
<input id="searchInput" name="search" type="text" accesskey="f" value="" />
 +
<input type='submit' name="go" class="searchButton" id="searchGoButton" value="Go" />&nbsp;
 +
<input type='submit' name="fulltext" class="searchButton" value="Search" />
 +
</div></form>
 +
</div>
 +
</div>
 +
<div class="portlet" id="p-tb">
 +
<h5>Toolbox</h5>
 +
<div class="pBody">
 +
<ul>
 +
<li id="t-whatlinkshere"><a href="/research/wiki/index.php/Special:Whatlinkshere/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">What links here</a></li>
 +
<li id="t-recentchangeslinked"><a href="/research/wiki/index.php/Special:Recentchangeslinked/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">Related changes</a></li>
 +
<li id="t-upload"><a href="/research/wiki/index.php/Special:Upload">Upload file</a></li>
 +
<li id="t-specialpages"><a href="/research/wiki/index.php/Special:Specialpages">Special pages</a></li>
 +
<li id="t-print"><a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&amp;printable=yes">Printable version</a></li> <li id="t-permalink"><a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&amp;oldid=4799">Permanent link</a></li> </ul>
 +
</div>
 +
</div>
 +
</div><!-- end of the left (by default at least) column -->
 +
<div class="visualClear"></div>
 +
<div id="footer">
 +
<div id="f-poweredbyico"><a href="http://www.mediawiki.org/"><img src="/research/wiki/skins/common/images/poweredby_mediawiki_88x31.png" alt="MediaWiki" /></a></div>
 +
<ul id="f-list">
 +
<li id="lastmod"> This page was last modified 15:26, 14 April 2007.</li>
 +
<li id="viewcount">This page has been accessed 17 times.</li>
 +
<li id="privacy"><a href="/research/wiki/index.php/Pslc:Privacy_policy" title="Pslc:Privacy policy">Privacy policy</a></li>
 +
<li id="about"><a href="/research/wiki/index.php/Pslc:About" title="Pslc:About">About Pslc</a></li>
 +
<li id="disclaimer"><a href="/research/wiki/index.php/Pslc:General_disclaimer" title="Pslc:General disclaimer">Disclaimers</a></li>
 +
</ul>
 +
</div>
 +
<script type="text/javascript"> if (window.runOnloadHook) runOnloadHook();</script>
 +
</div>
 +
<!-- Served by einstein.pslc.cs.cmu.edu in 0.16 secs. -->
 +
</body></html>

Revision as of 22:21, 25 July 2007

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <meta name="keywords" content=" REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions (Summer 2007) " /> <link rel="shortcut icon" href="/favicon.ico" /> <title> REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions (Summer 2007) - Pslc</title> <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "/research/wiki/skins/monobook/main.css?7"; /*]]>*/</style> <link rel="stylesheet" type="text/css" media="print" href="/research/wiki/skins/common/commonPrint.css" /> <script type="text/javascript">var skin = 'monobook';var stylepath = '/research/wiki/skins';</script> <script type="text/javascript" src="/research/wiki/skins/common/wikibits.js"></script> <script type="text/javascript" src="/research/wiki/index.php?title=-&action=raw&smaxage=0&gen=js"></script> <style type="text/css">/*<![CDATA[*/ @import "/research/wiki/index.php?title=MediaWiki:Common.css&action=raw&ctype=text/css&smaxage=18000"; @import "/research/wiki/index.php?title=MediaWiki:Monobook.css&action=raw&ctype=text/css&smaxage=18000"; @import "/research/wiki/index.php?title=-&action=raw&gen=css&maxage=18000"; /*]]>*/</style> </head> <body class="ns-0 ltr">

<a name="top" id="top"></a>

REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions (Summer 2007)

From Pslc

Jump to: <a href="#column-one">navigation</a>, <a href="#searchInput">search</a>

Contents

  • <a href="# Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions">1 REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions </a>
    • <a href="#Logistical_Information">1.1 Logistical Information</a>
    • <a href="#Abstract">1.2 Abstract</a>
    • <a href="#Glossary">1.3 Glossary</a>
    • <a href="#Research_question">1.4 Research question</a>
    • <a href="#Dependent_variables">1.5 Dependent variables</a>
    • <a href="#Independent_variables">1.6 Independent variables</a>
    • <a href="#Hypotheses">1.7 Hypotheses</a>
    • <a href="#Explanation">1.8 Explanation</a>
    • <a href="#Descendents">1.9 Descendents</a>
    • <a href="#Notes">1.10 Notes</a>
    • <a href="#Annotated_bibliography">1.11 Annotated bibliography</a>
<script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script>
[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions%28Summer_2007%29&action=edit&section=1" title="Edit section: REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions ">edit</a>]
<a name=" REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions "></a>

REAP Study on the Correlation Between Automatically and Manually Generated Reading Comprehension Questions

[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&action=edit&section=2" title="Edit section: Logistical Information">edit</a>]
<a name="Logistical_Information"></a>

Logistical Information

Contributors Christine M. Feeney and Michael Heilman
Study Start Date June, 2007
Study End Date July, 2007
Learnlab Courses none
Number of Students 30
Total Participant Hours (est.) 300
Data in Datashop no
[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&action=edit&section=3" title="Edit section: Abstract">edit</a>]
<a name="Abstract"></a>

Abstract

Previous psychological research has identified two types of comprehension: shallow, in which people can reproduce information, and deep, in which people can comprehend the meaning of information. In addition, researchers have found that these two types of comprehension are separate and disparate. The PSLC currently employs an English as a Second Language vocabulary tutor, REAP, that uses shallow, automatically generated, comprehension questions to check that students are actively reading, rather than just skimming, practice reading passages. The purpose of the current study was to examine performance on the REAP-type questions with manually authored, deeper, reading comprehension questions. Participants were thirty undergraduate students (male = 11) participating in summer research programs at Carnegie Mellon University. The researcher predicted a positive correlation between the two testing types, which was supported by the data (r = .366, p < 0.0005, one-tailed t-test).

[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&action=edit&section=4" title="Edit section: Glossary">edit</a>]
<a name="Glossary"></a>

Glossary

Shallow Comprehension: Processing is directed towards reproducing the learning material without necessarily understanding it.

Deep Comprehension: Processing is directed towards comprehending the intended meaning of the learning material.

[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%%28Summer_2007%29&action=edit&section=5" title="Edit section: Research question">edit</a>]
<a name="Research_question"></a>

Research question

Does performance on the shallow, automatically-generated questions correlate with performance on more sophisticated, deep reading comprehension questions?

[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&action=edit&section=6" title="Edit section: Dependent variables">edit</a>]
<a name="Dependent_variables"></a>

Dependent variables

Percentage of Shallow Comprehension Questions Answered Correctly.

Percentage of Deep Comprehension Questions Answered Correctly.

[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&action=edit&section=7" title="Edit section: Independent variables">edit</a>]
<a name="Independent_variables"></a>

Independent variable

Five reading passages, each of which was followed by four shallow reading comprehension questions and three to five deep reading comprehension questions.

[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&action=edit&section=8" title="Edit section: Hypotheses">edit</a>]
<a name="Hypotheses"></a>

Hypotheses

A significant positive correlation between shallow and deep reading comprehension questions will occur in this study.

[<a href="/research/wiki/index.php?title= REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_%28Summer_2007%29&action=edit&section=9" title="Edit section: Explanation">edit</a>]
<a name="Explanation"></a>

Explanation

The study found no significant differences between the two conditions (2 or 4 words/text). This null result may be due to statistical power and mistakes in the study design. The tutor had difficulty providing high quality readings with four vocabulary words. Thus, the tutor often had to give texts with 3 or even 2 words to students in the 4-word condition. Additionally, the optimal number of words per text (the density of practice) may be higher than 4 words. There may be a threshold at which the density of practice becomes too high. This threshold may actually be at 5 or even 10 words given the average length of the texts used by the REAP tutor. Some research suggests that up to 5% of the words in a text can be unknown before comprehension is impeded (Laufer, 1992). Although the task for that study was different (reading comprehension rather than vocabulary acquisition), it suggests that the optimal level of stretch may be such that as many as 20 words may be unknown in a text with 400 word types (which is approximately the mean number of word types in REAP texts). Other research, however, suggests higher thresholds up to 98% (Nation, 2001).

Although the results for stretch from this study were inconclusive, a great deal of interesting data were gathered. Some interesting results have been found, and further analysis is possible.

Many potential variables exist that could lead to successful learning of new vocabulary: general ESL ability as measured by the MTELP, time on task, number of words looked up, pre-test scores on the academic word list. A step-wise linear regression showed that the most significant predictor of acquisition of new vocabulary was the size of the student's vocabulary scores on the academic word list prior to instruction (r= 0.77, p < .0001). However, the effect of this prior knowledge was diminished by the number of texts the students read r= -0.17). Together, in the stepwise regression, these two variable accounted for 67% of the variance (r2= .67).

This finding corroborates findings of Stanovich (1986) and James (1996) who refer to the phenomenon as the "Matthew effect" (after a Bible passage), whereby the rich get richer and the poor get poorer. In this case, it is not 'general richness' expressed in terms of higher general ESL proficiency, but specifically knowledge of words on the academic word list. In other words, the more knowledge components in the domain of acquisition a learner brings to the task, the greater the potential for future learning.

It was also found that students frequently access dictionary definitions for non-target words that are well below their expected reading level (e.g., "hitting"). This may lead to revisions of curricula for ESL vocabulary learning.

[<a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&action=edit&section=10" title="Edit section: Descendents">edit</a>]
<a name="Descendents"></a>

Descendents

[<a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&action=edit&section=11" title="Edit section: Notes">edit</a>]
<a name="Notes"></a>

Notes

Target vocabulary lists were determined by a lengthy pre-test using multiple choice cloze questions. Each question takes 30-60 seconds on average, depending on the student. In later studies, self-assessment pre-tests were preferred due to time constraints and initial impressions of the students. Lengthy pre-tests for determining target word lists leave a poor initial impression of the tutor.

[<a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&action=edit&section=12" title="Edit section: Annotated bibliography">edit</a>]
<a name="Annotated_bibliography"></a>

Annotated bibliography

Laufer, B. (1992). How much lexis is necessary for reading comprehension? In Vocabulary and Applied Linguistics, Pierre J. L. Arnaud and Henri Béjoint (eds.), 126–132. London: Macmillan.

Retrieved from "<a href="http://www.learnlab.org/research/wiki/index.php/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">http://www.learnlab.org/research/wiki/index.php/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29</a>"
Views
  • <a href="/research/wiki/index.php/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">Article</a>
  • <a href="/research/wiki/index.php?title=Talk:REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&action=edit">Discussion</a>
  • <a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&action=edit">Edit</a>
  • <a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&action=history">History</a>
  • <a href="/research/wiki/index.php/Special:Movepage/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">Move</a>
  • <a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&action=watch">Watch</a>
Personal tools
  • <a href="/research/wiki/index.php/User:Feeney" class="new">Feeney</a>
  • <a href="/research/wiki/index.php/User_talk:Feeney" class="new">My talk</a>
  • <a href="/research/wiki/index.php/Special:Preferences">Preferences</a>
  • <a href="/research/wiki/index.php/Special:Watchlist">My watchlist</a>
  • <a href="/research/wiki/index.php/Special:Contributions/Feeney">My contributions</a>
  • <a href="/research/wiki/index.php?title=Special:Userlogout&returnto=REAP_Study_on_Vocabulary_Stretch_(Spring_2006)">Log out</a>

<script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script>

Navigation
  • <a href="/research/wiki/index.php/Main_Page">Main Page</a>
  • <a href="/research/wiki/index.php/Root_node">Root Node</a>
  • <a href="/research/wiki/index.php/Category:Glossary">Glossary</a>
  • <a href="/research/wiki/index.php/Special:Recentchanges">Recent changes</a>
  • <a href="/research/wiki/index.php/Special:Random">Random page</a>
  • <a href="/research/wiki/index.php/Help:Contents">Help</a>
clusters
  • <a href="/research/wiki/index.php/Coordinative_Learning">Coordinative Learning</a>
  • <a href="/research/wiki/index.php/Interactive_Communication">Interactive Communication</a>
  • <a href="/research/wiki/index.php/Refinement_and_Fluency">Refinement and Fluency</a>
courses
  • <a href="/research/wiki/index.php/Algebra">Algebra</a>
  • <a href="/research/wiki/index.php/Geometry">Geometry</a>
  • <a href="/research/wiki/index.php/Chinese">Chinese</a>
  • <a href="/research/wiki/index.php/French">French</a>
  • <a href="/research/wiki/index.php/English">English</a>
  • <a href="/research/wiki/index.php/Chemistry">Chemistry</a>
  • <a href="/research/wiki/index.php/Physics">Physics</a>
datashop
  • <a href="/research/wiki/index.php/DataShop_Pipeline">DataShop Pipeline</a>
Toolbox
  • <a href="/research/wiki/index.php/Special:Whatlinkshere/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">What links here</a>
  • <a href="/research/wiki/index.php/Special:Recentchangeslinked/REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29">Related changes</a>
  • <a href="/research/wiki/index.php/Special:Upload">Upload file</a>
  • <a href="/research/wiki/index.php/Special:Specialpages">Special pages</a>
  • <a href="/research/wiki/index.php?title=REAP_Study_on_Vocabulary_Stretch_%28Spring_2006%29&printable=yes">Printable version</a>

<script type="text/javascript"> if (window.runOnloadHook) runOnloadHook();</script>

</body></html>

Retrieved from "https://learnlab.org/wiki/index.php?title=REAP_Study_on_the_Correlation_Between_Automatically_and_Manually_Generated_Reading_Comprehension_Questions_(Summer_2007)&oldid=5877"