This is an automated email from the ASF dual-hosted git repository.
pgil pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ofbiz-framework.git
The following commit(s) were added to refs/heads/trunk by this push:
new d7a1b40d1f Improved: Add delegator method to create in mass many
entities (OFBIZ-13176) (#857)
d7a1b40d1f is described below
commit d7a1b40d1f5243cf38b2d755efcea12058e432f3
Author: Nicolas Malin <nicolas.ma...@nereide.fr>
AuthorDate: Fri Nov 15 14:43:03 2024 +0100
Improved: Add delegator method to create in mass many entities
(OFBIZ-13176) (#857)
Improved: Add delegator method to create in mass many entities (OFBIZ-13176)
When you need to load huge record on your database (like system
synchronization, data import) OFBiz can have latency with the delegator.create.
Incriminated unitary workflow and eeca control. You can disable eeca on use
a dedicate delegator but sometime it's not enough.
So for special case, we implement a new function
delegator.createAllByBatchProcess that will use an insert with one transaction
through the standard sql method INSERT TO ... VALUES ... .
By the way, like the process isn't unitary, the eeca system can't run on
this batch, so this function must therefore be used with care
Improve test to validate multi entity creation by batch.
Thanks to Néréide team for this contribution
---------
Co-authored-by: Gil Portenseigne <gil.portensei...@nereide.fr>
---
.../java/org/apache/ofbiz/entity/Delegator.java | 28 ++++++++++++
.../org/apache/ofbiz/entity/GenericDelegator.java | 52 ++++++++++++++++++++++
.../apache/ofbiz/entity/datasource/GenericDAO.java | 52 ++++++++++++++++++++++
.../ofbiz/entity/datasource/GenericHelper.java | 7 +++
.../ofbiz/entity/datasource/GenericHelperDAO.java | 14 ++++++
.../ofbiz/entity/datasource/ReadOnlyHelperDAO.java | 8 ++++
.../org/apache/ofbiz/entity/jdbc/SQLProcessor.java | 24 ++++++++++
.../apache/ofbiz/entity/test/EntityTestSuite.java | 19 ++++++++
8 files changed, 204 insertions(+)
diff --git
a/framework/entity/src/main/java/org/apache/ofbiz/entity/Delegator.java
b/framework/entity/src/main/java/org/apache/ofbiz/entity/Delegator.java
index 73da59c8a7..bdd7a91252 100644
--- a/framework/entity/src/main/java/org/apache/ofbiz/entity/Delegator.java
+++ b/framework/entity/src/main/java/org/apache/ofbiz/entity/Delegator.java
@@ -197,6 +197,34 @@ public interface Delegator {
*/
GenericValue createSingle(String entityName, Object singlePkValue) throws
GenericEntityException;
+ /**
+ * <p>Create the Entities from the List GenericValue instances to the
persistent
+ * store.</p>
+ * <p>This is different than the normal create method, because all creation
+ * will be done with one unique insert to go fast.</p>
+ * <p>For this reason eca can't be raised, so it's useful for process
+ * with huge data to inject into database</p>
+ * @param values
+ * List of GenericValue instances containing the entities to
create
+ */
+ void createAllByBatchProcess(List<GenericValue> values) throws
GenericEntityException;
+
+ /**
+ * <p>Create the Entities from the List GenericValue instances to the
persistent
+ * store.</p>
+ * <p>This is different than the normal create method, because all creation
+ * will be done with one unique insert to go fast.</p>
+ * <p>For this reason eca can't be raised, so it's useful for process
+ * with huge data to inject on database.</p>
+ * <p>As this is a huge process, we can specify whether we want to alert
the ofbiz cluster
+ * if we need to clean their cache or just wait the normal expiration</p>
+ * @param values
+ * List of GenericValue instances containing the entities to
create
+ * @param distribute
+ * set to true if we want to clean cache entity for ofbiz
cluster
+ */
+ void createAllByBatchProcess(List<GenericValue> values, boolean
distribute) throws GenericEntityException;
+
Object decryptFieldValue(String entityName, ModelField.EncryptMethod
encryptMethod, String encValue) throws EntityCryptoException;
Object encryptFieldValue(String entityName, ModelField.EncryptMethod
encryptMethod, Object fieldValue) throws EntityCryptoException;
diff --git
a/framework/entity/src/main/java/org/apache/ofbiz/entity/GenericDelegator.java
b/framework/entity/src/main/java/org/apache/ofbiz/entity/GenericDelegator.java
index ea2e46438d..2c0f34efde 100644
---
a/framework/entity/src/main/java/org/apache/ofbiz/entity/GenericDelegator.java
+++
b/framework/entity/src/main/java/org/apache/ofbiz/entity/GenericDelegator.java
@@ -39,6 +39,7 @@ import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.stream.Collectors;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.ofbiz.base.concurrent.ConstantFuture;
@@ -922,6 +923,57 @@ public class GenericDelegator implements Delegator {
}
}
+ /* (non-Javadoc)
+ * @see
org.apache.ofbiz.entity.Delegator#createAllByBatchProcess(org.apache.ofbiz.entity.Delegator)
+ */
+ @Override
+ public void createAllByBatchProcess(List<GenericValue> values, boolean
distribute) throws GenericEntityException {
+ boolean beganTransaction = ALWAYS_USE_TRANS
+ ? TransactionUtil.begin()
+ : false;
+
+ if (UtilValidate.isEmpty(values)) {
+ throw new GenericEntityException("No value to store");
+ }
+ Map<String, List<GenericValue>> sortedValuesByEntityName =
values.stream()
+ .collect(Collectors.groupingBy(GenericValue::getEntityName));
+ for (Map.Entry<String, List<GenericValue>> entry :
sortedValuesByEntityName.entrySet()) {
+ List<GenericValue> entityValues = entry.getValue();
+ String entityName = entry.getKey();
+ try {
+ GenericHelper helper = getEntityHelper(entityName);
+
+ helper.createAll(entityValues);
+
+ if (testMode) {
+ entityValues.forEach(v ->
+ storeForTestRollback(new
TestOperation(OperationType.INSERT, v)));
+ }
+ if (distribute) {
+ this.clearCacheLine(entityName);
+ }
+
+ TransactionUtil.commit(beganTransaction);
+ } catch (IllegalStateException | GenericEntityException e) {
+ String errMsg = String.format(
+ "Failure in create operation for list of entity on %s"
+ + " for %s elements with error : %s. Rolling
back transaction.",
+ entityName, entityValues.size(), e);
+ Debug.logError(errMsg, MODULE);
+ TransactionUtil.rollback(beganTransaction, errMsg, e);
+ throw new GenericEntityException(e);
+ }
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see
org.apache.ofbiz.entity.Delegator#createAllByBatchProcess(org.apache.ofbiz.entity.Delegator)
+ */
+ @Override
+ public void createAllByBatchProcess(List<GenericValue> values) throws
GenericEntityException {
+ createAllByBatchProcess(values, true);
+ }
+
/* (non-Javadoc)
* @see
org.apache.ofbiz.entity.Delegator#createOrStore(org.apache.ofbiz.entity.GenericValue)
*/
diff --git
a/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericDAO.java
b/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericDAO.java
index 2348ae96cb..404bb82d55 100644
---
a/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericDAO.java
+++
b/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericDAO.java
@@ -122,6 +122,58 @@ public class GenericDAO {
}
}
+ /**
+ * Insert in database all GenericValue of the same entity
+ * @param entities
+ * @return number of entity inserted in database
+ * @throws GenericEntityException
+ */
+ public int insertAll(List<GenericValue> entities) throws
GenericEntityException {
+ GenericEntity entity = entities.get(0);
+ ModelEntity modelEntity = entity.getModelEntity();
+
+ try (SQLProcessor sqlP = new SQLProcessor(entity.getDelegator(),
helperInfo)) {
+ try {
+ return multiInsert(entities, modelEntity,
modelEntity.getFieldsUnmodifiable(), sqlP);
+ } catch (GenericEntityException e) {
+ sqlP.rollback();
+ throw e;
+ }
+ }
+ }
+
+ private int multiInsert(List<GenericValue> entities, ModelEntity
modelEntity, List<ModelField> fieldsToSave,
+ SQLProcessor sqlP) throws GenericEntityException {
+ if (modelEntity instanceof ModelViewEntity) {
+ throw new GenericEntityException("Not Implemented");
+ }
+
+ StringBuilder sqlB = new StringBuilder("INSERT INTO ")
+ .append(modelEntity.getTableName(datasource))
+ .append(" (");
+
+ modelEntity.colNameString(fieldsToSave, sqlB, "");
+ sqlB.append(") VALUES (");
+ modelEntity.fieldsStringList(fieldsToSave, sqlB, "?", ", ");
+ sqlB.append(")");
+
+ try {
+ sqlP.prepareStatement(sqlB.toString());
+ for (GenericEntity ent : entities) {
+ SqlJdbcUtil.setValues(sqlP, fieldsToSave, ent,
modelFieldTypeReader);
+ sqlP.addBatch();
+ }
+ int retVal = sqlP.executeBatch();
+
+ for (GenericEntity ent : entities) {
+ ent.synchronizedWithDatasource();
+ }
+ return retVal;
+ } catch (GenericEntityException | SQLException e) {
+ throw new GenericEntityException("Error while inserting: " + sqlB,
e);
+ }
+ }
+
private int singleInsert(GenericEntity entity, ModelEntity modelEntity,
List<ModelField> fieldsToSave, SQLProcessor sqlP)
throws GenericEntityException {
if (modelEntity instanceof ModelViewEntity) {
diff --git
a/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericHelper.java
b/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericHelper.java
index 33020dfd1e..bc64d50ade 100644
---
a/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericHelper.java
+++
b/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericHelper.java
@@ -53,6 +53,13 @@ public interface GenericHelper {
*/
GenericValue create(GenericValue value) throws GenericEntityException;
+ /**
+ * Insert a given list of GenericValue to the database
+ * @param values
+ * @return The list of GenericValue created
+ */
+ List<GenericValue> createAll(List<GenericValue> values) throws
GenericEntityException;
+
/** Find a Generic Entity by its Primary Key
*@param primaryKey The primary key to find by.
*@return The GenericValue corresponding to the primaryKey
diff --git
a/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericHelperDAO.java
b/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericHelperDAO.java
index 2cc1e333eb..8ea09c59a5 100644
---
a/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericHelperDAO.java
+++
b/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/GenericHelperDAO.java
@@ -72,6 +72,20 @@ public class GenericHelperDAO implements GenericHelper {
return value;
}
+ /** Insert a given list of GenericValue to the database
+ *@return List of GenericValue instance created
+ */
+ public List<GenericValue> createAll(List<GenericValue> values) throws
GenericEntityException {
+ if (values.isEmpty()) {
+ return null;
+ }
+ int retVal = genericDAO.insertAll(values);
+ if (Debug.verboseOn()) {
+ Debug.logVerbose("Insert Return Value : " + retVal, MODULE);
+ }
+ return values;
+ }
+
/** Find a Generic Entity by its Primary Key
*@param primaryKey The primary key to find by.
*@return The GenericValue corresponding to the primaryKey
diff --git
a/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/ReadOnlyHelperDAO.java
b/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/ReadOnlyHelperDAO.java
index 0ab31dd337..0511b31292 100644
---
a/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/ReadOnlyHelperDAO.java
+++
b/framework/entity/src/main/java/org/apache/ofbiz/entity/datasource/ReadOnlyHelperDAO.java
@@ -64,6 +64,14 @@ public class ReadOnlyHelperDAO implements GenericHelper {
return null;
}
+ /** Read only, no creation realize on the database
+ *@return null
+ */
+ @Override
+ public List<GenericValue> createAll(List<GenericValue> value) throws
GenericEntityException {
+ return null;
+ }
+
/** Find a Generic Entity by its Primary Key
*@param primaryKey The primary key to find by.
*@return The GenericValue corresponding to the primaryKey
diff --git
a/framework/entity/src/main/java/org/apache/ofbiz/entity/jdbc/SQLProcessor.java
b/framework/entity/src/main/java/org/apache/ofbiz/entity/jdbc/SQLProcessor.java
index 30c2a7f2f1..d3afd8d326 100644
---
a/framework/entity/src/main/java/org/apache/ofbiz/entity/jdbc/SQLProcessor.java
+++
b/framework/entity/src/main/java/org/apache/ofbiz/entity/jdbc/SQLProcessor.java
@@ -33,6 +33,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.ofbiz.base.util.Debug;
@@ -858,4 +859,27 @@ public class SQLProcessor implements AutoCloseable {
TransactionUtil.printAllThreadsTransactionBeginStacks();
}
}
+
+ /**
+ * Ask the processor to execute the batch and return the number of rows
updated
+ * @return The number of rows updated
+ * @throws GenericDataSourceException
+ */
+ public int executeBatch() throws GenericDataSourceException {
+ try {
+ return Arrays.stream(ps.executeBatch()).sum();
+ } catch (SQLException sqle) {
+ this.checkLockWaitInfo(sqle);
+ throw new GenericDataSourceException("SQL Exception while
executing the following:" + sql, sqle);
+ }
+ }
+
+ /**
+ * Add to the processor a batch treatment
+ * @throws SQLException
+ */
+ public void addBatch() throws SQLException {
+ this.ind = 1;
+ this.getPreparedStatement().addBatch();
+ }
}
diff --git
a/framework/entity/src/main/java/org/apache/ofbiz/entity/test/EntityTestSuite.java
b/framework/entity/src/main/java/org/apache/ofbiz/entity/test/EntityTestSuite.java
index b95fc8e85d..19e1664367 100644
---
a/framework/entity/src/main/java/org/apache/ofbiz/entity/test/EntityTestSuite.java
+++
b/framework/entity/src/main/java/org/apache/ofbiz/entity/test/EntityTestSuite.java
@@ -184,6 +184,25 @@ public class EntityTestSuite extends EntityTestCase {
assertEquals("Finding removed value returns null", null, testValue);
}
+ /**
+ * Test to load huge entity
+ * @throws Exception the exception
+ */
+ public void testCreateAllValues() throws Exception {
+ List<GenericValue> testValues = new ArrayList<>(100);
+ for (int i = 0; i < 100; i++) {
+ testValues.add(getDelegator().makeValue("TestingType",
+ "testingTypeId", "TEST_CREATE_ALL" + i,
+ "description", "Testing Type #CreateAll" + i));
+ testValues.add(getDelegator().makeValue("Testing", "testingId",
"TEST_CREATE_DIST" + i));
+ }
+ getDelegator().createAllByBatchProcess(testValues);
+ assertEquals("create all insert 100 elements", 100,
getDelegator().findCountByCondition("TestingType",
+ EntityCondition.makeCondition("testingTypeId",
EntityOperator.LIKE, "TEST_CREATE_ALL%"), null, null));
+ assertEquals("create all insert 100 elements", 100,
getDelegator().findCountByCondition("Testing",
+ EntityCondition.makeCondition("testingId",
EntityOperator.LIKE, "TEST_CREATE_DIST%"), null, null));
+ }
+
/**
* Tests the entity cache
* @throws Exception the exception
