Author: rahul
Date: Sat Feb 7 06:23:06 2009
New Revision: 741845
URL: http://svn.apache.org/viewvc?rev=741845&view=rev
Log:
Stray line breaks in pipeline_basics tutorial code examples.
Patch by Ken Tanaka <Ken DOT Tanaka AT noaa DOT gov>
SANDBOX-278
Modified:
commons/sandbox/pipeline/trunk/src/site/xdoc/pipeline_basics.xml
Modified: commons/sandbox/pipeline/trunk/src/site/xdoc/pipeline_basics.xml
URL:
http://svn.apache.org/viewvc/commons/sandbox/pipeline/trunk/src/site/xdoc/pipeline_basics.xml?rev=741845&r1=741844&r2=741845&view=diff
==============================================================================
--- commons/sandbox/pipeline/trunk/src/site/xdoc/pipeline_basics.xml (original)
+++ commons/sandbox/pipeline/trunk/src/site/xdoc/pipeline_basics.xml Sat Feb 7
06:23:06 2009
@@ -477,27 +477,27 @@
Description: A sample configuration file for a very simple pipeline
--></span>
-<<b>pipeline</b>>
+<<span style="font-weight:bold;">pipeline</span>>
- <<b>driverFactory</b>
className="org.apache.commons.pipeline.driver.<b>DedicatedThreadStageDriverFactory</b>"
- id="<span
style="color:#008080;"><b>driverFactory</b></span>"/>
+ <<span style="font-weight:bold;">driverFactory</span>
className="org.apache.commons.pipeline.driver.<span
style="font-weight:bold;">DedicatedThreadStageDriverFactory</span>"
+ id="<span style="color:#008080;"><span
style="font-weight:bold;">driverFactory</span></span>"/>
<!--
((1)) The first stage recursively searches the directory given in the
feed statement.
The filePattern given will match any files beginning with
"HelloWorld".
-->
- <span style="color:#990000;"><<b>stage</b>
className="org.apache.commons.pipeline.stage.<b>FileFinderStage</b>"
- driverFactoryId="<span
style="color:#008080;"><b>driverFactory</b></span>"
- <span
style="color:#FF6600;">filePattern="<b>HelloWorld.*</b>"</span>/></span>
<span style="color:#FF6600;"><!-- ((3)) --></span>
+ <span style="color:#990000;"><<span
style="font-weight:bold;">stage</span>
className="org.apache.commons.pipeline.stage.<span
style="font-weight:bold;">FileFinderStage</span>"
+ driverFactoryId="<span style="color:#008080;"><span
style="font-weight:bold;">driverFactory</span></span>"
+ <span style="color:#FF6600;">filePattern="<span
style="font-weight:bold;">HelloWorld.*</span>"</span>/></span> <span
style="color:#FF6600;"><!-- ((3)) --></span>
<!-- Starting directory for the first stage. -->
- <span style="color:#00CC00;"><<b>feed</b>>
- <value><b>/data/sample</b></value> <!-- ((4)) -->
+ <span style="color:#00CC00;"><<span
style="font-weight:bold;">feed</span>>
+ <value><span
style="font-weight:bold;">/data/sample</span></value> <!-- ((4)) -->
</feed></span>
<!-- ((2)) Report the files found. -->
- <span style="color:#0000FF;"><<b>stage</b>
className="org.apache.commons.pipeline.stage.<b>LogStage</b>"
- driverFactoryId="<span
style="color:#008080;"><b>driverFactory</b></span>" /></span>
+ <span style="color:#0000FF;"><<span
style="font-weight:bold;">stage</span>
className="org.apache.commons.pipeline.stage.<span
style="font-weight:bold;">LogStage</span>"
+ driverFactoryId="<span style="color:#008080;"><span
style="font-weight:bold;">driverFactory</span></span>" /></span>
</pipeline></pre>
</td></tr></table>
@@ -507,7 +507,7 @@
"driverFactory", and this value is used by the
driverFactoryId in both
stages.
</p>
- <p><br /><br />
+ <p><br />
In theory a pipeline could consist of just one stage, but this
degenerate case
is not much different from a plain program except that it can
be easily expanded
with additional stages.
@@ -531,14 +531,14 @@
puts data into a database.
--></span>
-<<b>pipeline</b>>
+<<span style="font-weight:bold;">pipeline</span>>
- <<b>driverFactory</b>
className="org.apache.commons.pipeline.driver.<b>DedicatedThreadStageDriverFactory</b>"
- id="<span style="color:#808000;"><b>df0</b></span>"/>
+ <<span style="font-weight:bold;">driverFactory</span>
className="org.apache.commons.pipeline.driver.<span
style="font-weight:bold;">DedicatedThreadStageDriverFactory</span>"
+ id="<span style="color:#808000;"><span
style="font-weight:bold;">df0</span></span>"/>
- <<b>driverFactory</b>
className="org.apache.commons.pipeline.driver.<b>DedicatedThreadStageDriverFactory</b>"
- id="<span style="color:#008080;"><b>df1</b></span>">
+ <<span style="font-weight:bold;">driverFactory</span>
className="org.apache.commons.pipeline.driver.<span
style="font-weight:bold;">DedicatedThreadStageDriverFactory</span>"
+ id="<span style="color:#008080;"><span
style="font-weight:bold;">df1</span></span>">
<property propName="queueFactory"
className="org.apache.commons.pipeline.util.BlockingQueueFactory$ArrayBlockingQueueFactory"
capacity="4" fair="false"/>
@@ -550,8 +550,8 @@
In this instance almost all of the stages need a key to tell them what
type of data
to process.
-->
- <span style="color:#006B6B;"><<b>env</b>>
- <value key="<b>division</b>"><b>West</b></value>
<!-- ((9)) -->
+ <span style="color:#006B6B;"><<span
style="font-weight:bold;">env</span>>
+ <value key="<span
style="font-weight:bold;">division</span>"><span
style="font-weight:bold;">West</span></value> <!-- ((9)) -->
</env></span>
@@ -563,12 +563,12 @@
The filePattern in the stage block is the pattern to look for within
that directory.
-->
- <span style="color:#990000;"><<b>stage</b>
className="org.apache.commons.pipeline.stage.<b>FileFinderStage</b>"
- driverFactoryId="<span
style="color:#808000;"><b>df0</b></span>"
- <span
style="color:#FF6600;">filePattern="<b>SALES\.(ASWK|ST(GD|GL|LD))\.N.?\.D\d{5}</b>"</span>/></span>
<span style="color:#FF6600;"><!-- ((8)) --></span>
+ <span style="color:#990000;"><<span
style="font-weight:bold;">stage</span>
className="org.apache.commons.pipeline.stage.<span
style="font-weight:bold;">FileFinderStage</span>"
+ driverFactoryId="<span style="color:#808000;"><span
style="font-weight:bold;">df0</span></span>"
+ <span style="color:#FF6600;">filePattern="<span
style="font-weight:bold;">SALES\.(ASWK|ST(GD|GL|LD))\.N.?\.D\d{5}</span>"</span>/></span>
<span style="color:#FF6600;"><!-- ((8)) --></span>
- <span style="color:#00CC00;"><<b>feed</b>>
- <value><b>/data/INPUT/raw</b></value> <!-- ((7)),
((11)) -->
+ <span style="color:#00CC00;"><<span
style="font-weight:bold;">feed</span>>
+ <value><span
style="font-weight:bold;">/data/INPUT/raw</span></value> <!-- ((7)),
((11)) -->
</feed></span>
@@ -581,8 +581,8 @@
terminating processing. Zero (0) has the special meaning that ALL
available
files should be processed.
-->
- <span style="color:#0000FF;"><<b>stage</b>
className="com.demo.pipeline.stages.<b>FileSorterStage</b>"
- driverFactoryId="<span
style="color:#008080;"><b>df1</b></span>"
+ <span style="color:#0000FF;"><<span
style="font-weight:bold;">stage</span>
className="com.demo.pipeline.stages.<span
style="font-weight:bold;">FileSorterStage</span>"
+ driverFactoryId="<span style="color:#008080;"><span
style="font-weight:bold;">df1</span></span>"
filesToProcess="0"/></span>
@@ -594,16 +594,16 @@
WARNING: The value for htmlPipelineKey in the stage declaration here
must exactly match the branch pipeline key further down in this file.
-->
- <span style="color:#9900CC;"><<b>stage</b>
className="com.demo.pipeline.stages.<b>FileReaderStage</b>"
- driverFactoryId="<span
style="color:#008080;"><b>df1</b></span>"
- htmlPipelineKey="<span
style="color:#FF00FF;"><b>sales2html</b></span>"/></span>
+ <span style="color:#9900CC;"><<span
style="font-weight:bold;">stage</span>
className="com.demo.pipeline.stages.<span
style="font-weight:bold;">FileReaderStage</span>"
+ driverFactoryId="<span style="color:#008080;"><span
style="font-weight:bold;">df1</span></span>"
+ htmlPipelineKey="<span style="color:#FF00FF;"><span
style="font-weight:bold;">sales2html</span></span>"/></span>
<!--
((4)) Write the data from the FileReaderStage stage into the database.
-->
- <span style="color:#CC6633;"><<b>stage</b>
className="com.demo.pipeline.stages.<b>DatabaseWriterStage</b>"
- driverFactoryId="<span
style="color:#008080;"><b>df1</b></span>">
+ <span style="color:#CC6633;"><<span
style="font-weight:bold;">stage</span>
className="com.demo.pipeline.stages.<span
style="font-weight:bold;">DatabaseWriterStage</span>"
+ driverFactoryId="<span style="color:#008080;"><span
style="font-weight:bold;">df1</span></span>">
<datasource user="test"
password="abc123"
@@ -614,7 +614,7 @@
<database-proxy
className="gov.noaa.gdsg.sql.oracle.OracleDatabaseProxy" />
- <tablePath path="<span
style="color:#339933;"><b>summary.inventory</b></span>" /> <span
style="color:#339933;"><!-- ((13)) --></span>
+ <tablePath path="<span style="color:#339933;"><span
style="font-weight:bold;">summary.inventory</span></span>" /> <span
style="color:#339933;"><!-- ((13)) --></span>
</stage></span>
@@ -626,15 +626,15 @@
WARNING: The value for the branch pipeline key declaration here must
exactly match the htmlPipelineKey in the FileReaderStage stage in this
file.
-->
- <span style="color:#FF00FF;"><<b>branch</b>>
- <<b>pipeline</b> key="<b>sales2html</b>"> <!--
((10)) --></span>
+ <span style="color:#FF00FF;"><<span
style="font-weight:bold;">branch</span>>
+ <<span style="font-weight:bold;">pipeline</span> key="<span
style="font-weight:bold;">sales2html</span>"> <!-- ((10))
--></span>
- <span style="color:#006B6B;"><<b>env</b>>
- <value
key="<b>division</b>"><b>West</b></value> <!-- ((14))
-->
+ <span style="color:#006B6B;"><<span
style="font-weight:bold;">env</span>>
+ <value key="<span
style="font-weight:bold;">division</span>"><span
style="font-weight:bold;">West</span></value> <!-- ((14)) -->
</env></span>
- <<b>driverFactory</b>
className="org.apache.commons.pipeline.driver.<b>DedicatedThreadStageDriverFactory</b>"
- id="<span
style="color:#EB613D;"><b>df2</b></span>">
+ <<span style="font-weight:bold;">driverFactory</span>
className="org.apache.commons.pipeline.driver.<span
style="font-weight:bold;">DedicatedThreadStageDriverFactory</span>"
+ id="<span style="color:#EB613D;"><span
style="font-weight:bold;">df2</span></span>">
<property propName="queueFactory"
className="org.apache.commons.pipeline.util.BlockingQueueFactory$ArrayBlockingQueueFactory"
capacity="4" fair="false"/>
@@ -642,15 +642,15 @@
<!-- ((5)) HTMLWriterStage -->
- <span style="color:#009900;"><<b>stage</b>
className="com.demo.pipeline.stages.<b>HTMLWriterStage</b>"
- driverFactoryId="<span
style="color:#EB613D;"><b>df2</b></span>"
- <span
style="color:#660000;">outputFilePath="<b>/data/OUTPUT/web</b>"/>
<!-- ((12)) --></span></span>
+ <span style="color:#009900;"><<span
style="font-weight:bold;">stage</span>
className="com.demo.pipeline.stages.<span
style="font-weight:bold;">HTMLWriterStage</span>"
+ driverFactoryId="<span style="color:#EB613D;"><span
style="font-weight:bold;">df2</span></span>"
+ <span style="color:#660000;">outputFilePath="<span
style="font-weight:bold;">/data/OUTPUT/web</span>"/> <!-- ((12))
--></span></span>
<!-- ((6)) StatPlotterStage -->
- <span style="color:#009900;"><<b>stage</b>
className="com.demo.pipeline.stages.<b>StatPlotterStage</b>"
- driverFactoryId="<span
style="color:#EB613D;"><b>df2</b></span>"
- <span
style="color:#660000;">outputFilePath="<b>/data/OUTPUT/web</b>"/>
<!-- ((12)) --></span></span>
+ <span style="color:#009900;"><<span
style="font-weight:bold;">stage</span>
className="com.demo.pipeline.stages.<span
style="font-weight:bold;">StatPlotterStage</span>"
+ driverFactoryId="<span style="color:#EB613D;"><span
style="font-weight:bold;">df2</span></span>"
+ <span style="color:#660000;">outputFilePath="<span
style="font-weight:bold;">/data/OUTPUT/web</span>"/> <!-- ((12))
--></span></span>
<span style="color:#FF00FF;"></pipeline>
</branch></span>
@@ -665,6 +665,17 @@
in both the main pipeline and the branch pipeline. This is
because
branches don't share the same environment constants.
</p>
+ <p>
+ The driverFactories "df1" and "df2"
override the default queueFactory
+ by specifying the <b>ArrayBlockingQueueFactory</b>. They do
this in order to
+ limit the queue sizes of the stages that use df1 or df2,
setting a <b>capacity</b> of 4
+ objects. This is often done to limit the resources used by the
pipeline, and
+ may be necessary if an unbounded queue is using all of the
available java
+ memory or exceeds the number of open filehandles allowed. The
queue size
+ cannot be changed after the queue is created. The <b>fair</b>
attribute can be
+ set to "false", since just one thread is accessing
the queue.
+ If fair = "true" then there is additional overhead
to make sure all
+ threads that access the queue are processed in order (requests
are FIFO).</p>
</subsection>
</section>
<section name="TODO">
