You're probably better off in this instance creating your own process based on SolrJ and your jdbc-driver-of-choice. DIH doesn't provide much in the way of fine-grained control over all aspects of the process, and at +30 hours I suspect you want some better control.
FWIW, SolrJ is not very hard at all to use for this kind of thing. Best Erick On Mon, Feb 7, 2011 at 10:59 AM, Mark <static.void....@gmail.com> wrote: > Typo in subject > > On 2/7/11 7:59 AM, Mark wrote: > >> I'm receiving the following exception when trying to perform a full-import >> (~30 hours). Any idea on ways I could fix this? >> >> Is there an easy way to use DIH to break apart a full-import into multiple >> pieces? IE 3 mini-imports instead of 1 large import? >> >> Thanks. >> >> >> >> >> Feb 7, 2011 5:52:33 AM org.apache.solr.handler.dataimport.JdbcDataSource >> closeConnection >> SEVERE: Ignoring Error when closing connection >> com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: >> Communications link failure during rollback(). Transaction resolution >> unknown. >> at sun.reflect.GeneratedConstructorAccessor27.newInstance(Unknown >> Source) >> at >> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) >> at java.lang.reflect.Constructor.newInstance(Constructor.java:532) >> at com.mysql.jdbc.Util.handleNewInstance(Util.java:407) >> at com.mysql.jdbc.Util.getInstance(Util.java:382) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1013) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:987) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:982) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:927) >> at com.mysql.jdbc.ConnectionImpl.rollback(ConnectionImpl.java:4751) >> at com.mysql.jdbc.ConnectionImpl.realClose(ConnectionImpl.java:4345) >> at com.mysql.jdbc.ConnectionImpl.close(ConnectionImpl.java:1564) >> at >> org.apache.solr.handler.dataimport.JdbcDataSource.closeConnection(JdbcDataSource.java:399) >> at >> org.apache.solr.handler.dataimport.JdbcDataSource.close(JdbcDataSource.java:390) >> at >> org.apache.solr.handler.dataimport.DataConfig$Entity.clearCache(DataConfig.java:174) >> at >> org.apache.solr.handler.dataimport.DataConfig$Entity.clearCache(DataConfig.java:165) >> at >> org.apache.solr.handler.dataimport.DataConfig.clearCaches(DataConfig.java:332) >> at >> org.apache.solr.handler.dataimport.DataImporter.doDeltaImport(DataImporter.java:360) >> at >> org.apache.solr.handler.dataimport.DataImporter.runCmd(DataImporter.java:391) >> at >> org.apache.solr.handler.dataimport.DataImporter$1.run(DataImporter.java:370) >> Feb 7, 2011 5:52:33 AM org.apache.solr.handler.dataimport.JdbcDataSource >> closeConnection >> SEVERE: Ignoring Error when closing connection >> java.sql.SQLException: Streaming result set >> com.mysql.jdbc.RowDataDynamic@1a797305 is still active. No statements may >> be issued when any streaming result sets are open and in use on a given >> connection. Ensure that you have called .close() on any active streaming >> result sets before attempting more queries. >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:934) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:931) >> at >> com.mysql.jdbc.MysqlIO.checkForOutstandingStreamingData(MysqlIO.java:2724) >> at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:1895) >> at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2140) >> at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2620) >> at >> com.mysql.jdbc.ConnectionImpl.rollbackNoChecks(ConnectionImpl.java:4854) >> at com.mysql.jdbc.ConnectionImpl.rollback(ConnectionImpl.java:4737) >> at com.mysql.jdbc.ConnectionImpl.realClose(ConnectionImpl.java:4345) >> at com.mysql.jdbc.ConnectionImpl.close(ConnectionImpl.java:1564) >> at >> org.apache.solr.handler.dataimport.JdbcDataSource.closeConnection(JdbcDataSource.java:399) >> at >> org.apache.solr.handler.dataimport.JdbcDataSource.close(JdbcDataSource.java:390) >> at >> org.apache.solr.handler.dataimport.DataConfig$Entity.clearCache(DataConfig.java:174) >> at >> org.apache.solr.handler.dataimport.DataConfig.clearCaches(DataConfig.java:332) >> at >> org.apache.solr.handler.dataimport.DataImporter.doDeltaImport(DataImporter.java:360) >> at >> org.apache.solr.handler.dataimport.DataImporter.runCmd(DataImporter.java:391) >> at >> org.apache.solr.handler.dataimport.DataImporter$1.run(DataImporter.java:370) >> Feb 7, 2011 7:03:29 AM org.apache.solr.handler.dataimport.JdbcDataSource >> closeConnection >> SEVERE: Ignoring Error when closing connection >> com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: >> Communications link failure during rollback(). Transaction resolution >> unknown. >> at sun.reflect.GeneratedConstructorAccessor27.newInstance(Unknown >> Source) >> at >> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) >> at java.lang.reflect.Constructor.newInstance(Constructor.java:532) >> at com.mysql.jdbc.Util.handleNewInstance(Util.java:407) >> at com.mysql.jdbc.Util.getInstance(Util.java:382) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1013) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:987) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:982) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:927) >> at com.mysql.jdbc.ConnectionImpl.rollback(ConnectionImpl.java:4751) >> at com.mysql.jdbc.ConnectionImpl.realClose(ConnectionImpl.java:4345) >> at com.mysql.jdbc.ConnectionImpl.close(ConnectionImpl.java:1564) >> at >> org.apache.solr.handler.dataimport.JdbcDataSource.closeConnection(JdbcDataSource.java:399) >> at >> org.apache.solr.handler.dataimport.JdbcDataSource.close(JdbcDataSource.java:390) >> at >> org.apache.solr.handler.dataimport.DataConfig$Entity.clearCache(DataConfig.java:174) >> at >> org.apache.solr.handler.dataimport.DataConfig.clearCaches(DataConfig.java:332) >> at >> org.apache.solr.handler.dataimport.DataImporter.doDeltaImport(DataImporter.java:360) >> at >> org.apache.solr.handler.dataimport.DataImporter.runCmd(DataImporter.java:391) >> at >> org.apache.solr.handler.dataimport.DataImporter$1.run(DataImporter.java:370) >> Feb 7, 2011 7:03:29 AM org.apache.solr.handler.dataimport.JdbcDataSource >> closeConnection >> SEVERE: Ignoring Error when closing connection >> com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: >> Communications link failure during rollback(). Transaction resolution >> unknown. >> at sun.reflect.GeneratedConstructorAccessor27.newInstance(Unknown >> Source) >> at >> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) >> at java.lang.reflect.Constructor.newInstance(Constructor.java:532) >> at com.mysql.jdbc.Util.handleNewInstance(Util.java:407) >> at com.mysql.jdbc.Util.getInstance(Util.java:382) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1013) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:987) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:982) >> at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:927) >> at com.mysql.jdbc.ConnectionImpl.rollback(ConnectionImpl.java:4751) >> at com.mysql.jdbc.ConnectionImpl.realClose(ConnectionImpl.java:4345) >> at com.mysql.jdbc.ConnectionImpl.close(ConnectionImpl.java:1564) >> at >> org.apache.solr.handler.dataimport.JdbcDataSource.closeConnection(JdbcDataSource.java:399) >> at >> org.apache.solr.handler.dataimport.JdbcDataSource.close(JdbcDataSource.java:390) >> at >> org.apache.solr.handler.dataimport.DataConfig$Entity.clearCache(DataConfig.java:174) >> at >> org.apache.solr.handler.dataimport.DataConfig$Entity.clearCache(DataConfig.java:165) >> at >> org.apache.solr.handler.dataimport.DataConfig.clearCaches(DataConfig.java:332) >> at >> org.apache.solr.handler.dataimport.DataImporter.doDeltaImport(DataImporter.java:360) >> at >> org.apache.solr.handler.dataimport.DataImporter.runCmd(DataImporter.java:391) >> at >> org.apache.solr.handler.dataimport.DataImporter$1.run(DataImporter.java:370) >> >>
