You are right - I just checked and now the other field
(vulnerable-software) that is using the same xpath has blank values.
BTW - It looks like this also works:
<field column="product" sourceColName="vulnerable-software"
commonField="false" regex=":" replaceWith=" "/>
Here are the results for one row in json:
"responseHeader":{
"status":0,
"QTime":1,
"params":{
"fl":"*",
"indent":"true",
"start":"0",
"q":"*:*",
"wt":"json",
"rows":"1"}},
"response":{"numFound":6717,"start":0,"docs":[
{
"product":["cpe /o freebsd freebsd 2.2.8",
"cpe /o freebsd freebsd 1.1.5.1",
"cpe /o freebsd freebsd 2.2.3",
"cpe /o freebsd freebsd 2.2.2",
"cpe /o freebsd freebsd 2.2.5",
"cpe /o freebsd freebsd 2.2.4",
"cpe /o freebsd freebsd 2.0.5",
"cpe /o freebsd freebsd 2.2.6",
"cpe /o freebsd freebsd 2.1.6.1",
"cpe /o freebsd freebsd 2.0.1",
"cpe /o freebsd freebsd 2.2",
"cpe /o freebsd freebsd 2.0",
"cpe /o openbsd openbsd 2.3",
"cpe /o freebsd freebsd 3.0",
"cpe /o freebsd freebsd 1.1",
"cpe /o freebsd freebsd 2.1.6",
"cpe /o openbsd openbsd 2.4",
"cpe /o bsdi bsd_os 3.1",
"cpe /o freebsd freebsd 1.0",
"cpe /o freebsd freebsd 2.1.7",
"cpe /o freebsd freebsd 1.2",
"cpe /o freebsd freebsd 2.1.5",
"cpe /o freebsd freebsd 2.1.7.1",
"cpe /o freebsd freebsd 2.2.8",
"cpe /o freebsd freebsd 1.1.5.1",
"cpe /o freebsd freebsd 2.2.3",
"cpe /o freebsd freebsd 2.2.2",
"cpe /o freebsd freebsd 2.2.5",
"cpe /o freebsd freebsd 2.2.4",
"cpe /o freebsd freebsd 2.0.5",
"cpe /o freebsd freebsd 2.2.6",
"cpe /o freebsd freebsd 2.1.6.1",
"cpe /o freebsd freebsd 2.0.1",
"cpe /o freebsd freebsd 2.2",
"cpe /o freebsd freebsd 2.0",
"cpe /o openbsd openbsd 2.3",
"cpe /o freebsd freebsd 3.0",
"cpe /o freebsd freebsd 1.1",
"cpe /o freebsd freebsd 2.1.6",
"cpe /o openbsd openbsd 2.4",
"cpe /o bsdi bsd_os 3.1",
"cpe /o freebsd freebsd 1.0",
"cpe /o freebsd freebsd 2.1.7",
"cpe /o freebsd freebsd 1.2",
"cpe /o freebsd freebsd 2.1.5",
"cpe /o freebsd freebsd 2.1.7.1"],
"id":"CVE-1999-0001",
"summary":"ip_input.c in BSD-derived TCP/IP implementations
allows remote attackers to cause a denial of service (crash or hang) via
crafted packets.",
"vulnerable-configuration":["cpe:/o:bsdi:bsd_os:3.1",
"cpe:/o:freebsd:freebsd:1.0",
"cpe:/o:freebsd:freebsd:1.1",
"cpe:/o:freebsd:freebsd:1.1.5.1",
"cpe:/o:freebsd:freebsd:1.2",
"cpe:/o:freebsd:freebsd:2.0",
"cpe:/o:freebsd:freebsd:2.0.5",
"cpe:/o:freebsd:freebsd:2.1.5",
"cpe:/o:freebsd:freebsd:2.1.6",
"cpe:/o:freebsd:freebsd:2.1.6.1",
"cpe:/o:freebsd:freebsd:2.1.7",
"cpe:/o:freebsd:freebsd:2.1.7.1",
"cpe:/o:freebsd:freebsd:2.2",
"cpe:/o:freebsd:freebsd:2.2.3",
"cpe:/o:freebsd:freebsd:2.2.4",
"cpe:/o:freebsd:freebsd:2.2.5",
"cpe:/o:freebsd:freebsd:2.2.6",
"cpe:/o:freebsd:freebsd:2.2.8",
"cpe:/o:freebsd:freebsd:3.0",
"cpe:/o:openbsd:openbsd:2.3",
"cpe:/o:openbsd:openbsd:2.4",
"cpe:/o:freebsd:freebsd:2.2.2",
"cpe:/o:freebsd:freebsd:2.0.1"],
"cve":"CVE-1999-0001",
"cwe":"CWE-20",
"published":"1999-12-30T00:00:00.000-05:00",
"vulnerable-software":["cpe:/o:freebsd:freebsd:2.2.8",
"cpe:/o:freebsd:freebsd:1.1.5.1",
"cpe:/o:freebsd:freebsd:2.2.3",
"cpe:/o:freebsd:freebsd:2.2.2",
"cpe:/o:freebsd:freebsd:2.2.5",
"cpe:/o:freebsd:freebsd:2.2.4",
"cpe:/o:freebsd:freebsd:2.0.5",
"cpe:/o:freebsd:freebsd:2.2.6",
"cpe:/o:freebsd:freebsd:2.1.6.1",
"cpe:/o:freebsd:freebsd:2.0.1",
"cpe:/o:freebsd:freebsd:2.2",
"cpe:/o:freebsd:freebsd:2.0",
"cpe:/o:openbsd:openbsd:2.3",
"cpe:/o:freebsd:freebsd:3.0",
"cpe:/o:freebsd:freebsd:1.1",
"cpe:/o:freebsd:freebsd:2.1.6",
"cpe:/o:openbsd:openbsd:2.4",
"cpe:/o:bsdi:bsd_os:3.1",
"cpe:/o:freebsd:freebsd:1.0",
"cpe:/o:freebsd:freebsd:2.1.7",
"cpe:/o:freebsd:freebsd:1.2",
"cpe:/o:freebsd:freebsd:2.1.5",
"cpe:/o:freebsd:freebsd:2.1.7.1"],
"modified":"2010-12-16T00:00:00.000-05:00",
"_version_":1491493094540967936}]
}}
On 1/27/15, 5:19 PM, Alexandre Rafalovitch wrote:
One xpath per field definition. You had two fields for the same xpath.
If they were the same value, the best bet would be to deal with it via
copyField in the schema.
No idea why regex thing makes a difference, are you sure the other
field is also still being indexed?
Regards,
Alex.
----
Sign up for my Solr resources newsletter at http://www.solr-start.com/
On 27 January 2015 at 17:11, Carl Roberts <carl.roberts.zap...@gmail.com> wrote:
Well - I got this to work. Noticed that when log4j is enabled product-info
was in the import as product-info=[], so I then played with the field and
got this definition to work in the rss-data-config.xml file:
<field column="product-info"
xpath="/nvd/entry/vulnerable-software-list/product" commonField="false"
regex=":" replaceWith=" "/>
Don't ask me why the other one didn't work, as I think it should have worked
also.
On 1/27/15, 3:42 PM, Carl Roberts wrote:
Hi,
I have tried to reindex to add a new field named product-info and no
matter what I do, I cannot get the new field to appear in the index after
import via DIH.
Here is the rss-data-config.xml configuration (field product-info is the
new field I added):
<dataConfig>
<dataSource type="ZIPURLDataSource" connectionTimeout="15000"
readTimeout="30000"/>
<document>
<entity name="cve-2002"
pk="id"
url="https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2002.xml.zip";
processor="XPathEntityProcessor"
forEach="/nvd/entry"
transformer="RegexTransformer">
<field column="id" xpath="/nvd/entry/@id" commonField="false"
/>
<field column="cve" xpath="/nvd/entry/cve-id"
commonField="false" />
<field column="cwe" xpath="/nvd/entry/cwe/@id"
commonField="false" />
*<field column="product-info"
xpath="/nvd/entry/vulnerable-software-list/product" commonField="false"/>*
<field column="vulnerable-configuration"
xpath="/nvd/entry/vulnerable-configuration/logical-test/fact-ref/@name"
commonField="false"/>
<field column="vulnerable-software"
xpath="/nvd/entry/vulnerable-software-list/product" commonField="false"/>
<field column="published"
xpath="/nvd/entry/published-datetime" commonField="false" />
<field column="modified"
xpath="/nvd/entry/last-modified-datetime" commonField="false" />
<field column="summary" xpath="/nvd/entry/summary"
commonField="false" />
</entity>
</document>
</dataConfig>
Here is the section that contains the new product-info field in
schema.xml:
<field name="text" type="text_general" indexed="true" stored="false"
multiValued="true"/>
<field name="_version_" type="long" indexed="true" stored="true"/>
<field name="id" type="text_general" indexed="true" stored="true"/>
<field name="cve" type="text_general" indexed="true" stored="true"/>
<field name="cwe" type="text_general" indexed="true" stored="true"/>
*<field name="product-info" type="text_general" indexed="true"
stored="true" multiValued="true"/>*
<field name="vulnerable-configuration" type="text_general"
indexed="true" stored="true" multiValued="true"/>
<field name="vulnerable-software" type="text_general" indexed="true"
stored="true" multiValued="true"/>
<field name="published" type="text_general" indexed="true"
stored="true" />
<field name="modified" type="text_general" indexed="true" stored="true"
/>
<field name="summary" type="text_general" indexed="true" stored="true"
/>
Field product-info is defined in the same manner as vulnerable-software
field and it pulls the same data as vulnerable-software list field via the
same xpath, yet vulnerable-software field shows up in the results and
product-info field does not. Here is the response for a query after the
import takes place - field product-info is missing:
~/dev/solr-4.10.3$ curl
"http://localhost:8983/solr/nvd-rss/select?wt=json&indent=true&q=*:*&start=0&&rows=1";
{
"responseHeader":{
"status":0,
"QTime":1,
"params":{
"indent":"true",
"start":"0",
"q":"*:*",
"wt":"json",
"rows":"1"}},
"response":{"numFound":6717,"start":0,"docs":[
{
"id":"CVE-1999-0001",
"summary":"ip_input.c in BSD-derived TCP/IP implementations allows
remote attackers to cause a denial of service (crash or hang) via crafted
packets.",
"vulnerable-configuration":["cpe:/o:bsdi:bsd_os:3.1",
"cpe:/o:freebsd:freebsd:1.0",
"cpe:/o:freebsd:freebsd:1.1",
"cpe:/o:freebsd:freebsd:1.1.5.1",
"cpe:/o:freebsd:freebsd:1.2",
"cpe:/o:freebsd:freebsd:2.0",
"cpe:/o:freebsd:freebsd:2.0.5",
"cpe:/o:freebsd:freebsd:2.1.5",
"cpe:/o:freebsd:freebsd:2.1.6",
"cpe:/o:freebsd:freebsd:2.1.6.1",
"cpe:/o:freebsd:freebsd:2.1.7",
"cpe:/o:freebsd:freebsd:2.1.7.1",
"cpe:/o:freebsd:freebsd:2.2",
"cpe:/o:freebsd:freebsd:2.2.3",
"cpe:/o:freebsd:freebsd:2.2.4",
"cpe:/o:freebsd:freebsd:2.2.5",
"cpe:/o:freebsd:freebsd:2.2.6",
"cpe:/o:freebsd:freebsd:2.2.8",
"cpe:/o:freebsd:freebsd:3.0",
"cpe:/o:openbsd:openbsd:2.3",
"cpe:/o:openbsd:openbsd:2.4",
"cpe:/o:freebsd:freebsd:2.2.2",
"cpe:/o:freebsd:freebsd:2.0.1"],
"cve":"CVE-1999-0001",
"cwe":"CWE-20",
"published":"1999-12-30T00:00:00.000-05:00",
"vulnerable-software":["cpe:/o:freebsd:freebsd:2.2.8",
"cpe:/o:freebsd:freebsd:1.1.5.1",
"cpe:/o:freebsd:freebsd:2.2.3",
"cpe:/o:freebsd:freebsd:2.2.2",
"cpe:/o:freebsd:freebsd:2.2.5",
"cpe:/o:freebsd:freebsd:2.2.4",
"cpe:/o:freebsd:freebsd:2.0.5",
"cpe:/o:freebsd:freebsd:2.2.6",
"cpe:/o:freebsd:freebsd:2.1.6.1",
"cpe:/o:freebsd:freebsd:2.0.1",
"cpe:/o:freebsd:freebsd:2.2",
"cpe:/o:freebsd:freebsd:2.0",
"cpe:/o:openbsd:openbsd:2.3",
"cpe:/o:freebsd:freebsd:3.0",
"cpe:/o:freebsd:freebsd:1.1",
"cpe:/o:freebsd:freebsd:2.1.6",
"cpe:/o:openbsd:openbsd:2.4",
"cpe:/o:bsdi:bsd_os:3.1",
"cpe:/o:freebsd:freebsd:1.0",
"cpe:/o:freebsd:freebsd:2.1.7",
"cpe:/o:freebsd:freebsd:1.2",
"cpe:/o:freebsd:freebsd:2.1.5",
"cpe:/o:freebsd:freebsd:2.1.7.1"],
"modified":"2010-12-16T00:00:00.000-05:00",
"_version_":1491484873657942016}]
}}
This is what I have tried so far:
Restarted Solr to reload the schema and reimported via full-import and
clean=true.
Invoked the following command to reload the schema and reimported via
full-import and clean=true
curl
"http://localhost:8983/solr/admin/cores?action=RELOAD&core=nvd-rss";
I am not sure why this is not working as everything seems correct to me in
my setup. Could this be a bug?
Regards,
Joe
