From 714b2a5f70abb85cd45131f0b930c91f0c67ccf7 Mon Sep 17 00:00:00 2001 From: Paul Guo Date: Thu, 28 Feb 2019 15:43:34 +0800 Subject: [PATCH v4] Multi insert in Create Table As. This could improve the performance and also benefits 'create materialized view' since that uses the code of create table as. --- src/backend/access/heap/heapam.c | 6 ++- src/backend/commands/copy.c | 24 +++------- src/backend/commands/createas.c | 81 ++++++++++++++++++++++++++------ src/include/access/heapam.h | 11 +++++ 4 files changed, 88 insertions(+), 34 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index e9544822bf..8a844b3b5f 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -2106,7 +2106,6 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, CommandId cid, int options, BulkInsertState bistate) { TransactionId xid = GetCurrentTransactionId(); - HeapTuple *heaptuples; int i; int ndone; PGAlignedBlock scratch; @@ -2115,6 +2114,10 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, Size saveFreeSpace; bool need_tuple_data = RelationIsLogicallyLogged(relation); bool need_cids = RelationIsAccessibleInLogicalDecoding(relation); + /* Declare it as static to let this memory be not on stack. */ + static HeapTuple heaptuples[MAX_MULTI_INSERT_TUPLES]; + + Assert(ntuples <= MAX_MULTI_INSERT_TUPLES); /* currently not needed (thus unsupported) for heap_multi_insert() */ AssertArg(!(options & HEAP_INSERT_NO_LOGICAL)); @@ -2124,7 +2127,6 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, HEAP_DEFAULT_FILLFACTOR); /* Toast and set header data in all the slots */ - heaptuples = palloc(ntuples * sizeof(HeapTuple)); for (i = 0; i < ntuples; i++) { HeapTuple tuple; diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 3aeef30b28..893c2031c2 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -233,18 +233,6 @@ typedef struct uint64 processed; /* # of tuples processed */ } DR_copy; - -/* - * No more than this many tuples per CopyMultiInsertBuffer - * - * Caution: Don't make this too big, as we could end up with this many - * CopyMultiInsertBuffer items stored in CopyMultiInsertInfo's - * multiInsertBuffers list. Increasing this can cause quadratic growth in - * memory requirements during copies into partitioned tables with a large - * number of partitions. - */ -#define MAX_BUFFERED_TUPLES 1000 - /* * Flush buffers if there are >= this many bytes, as counted by the input * size, of tuples stored. @@ -257,11 +245,11 @@ typedef struct /* Stores multi-insert data related to a single relation in CopyFrom. */ typedef struct CopyMultiInsertBuffer { - TupleTableSlot *slots[MAX_BUFFERED_TUPLES]; /* Array to store tuples */ + TupleTableSlot *slots[MAX_MULTI_INSERT_TUPLES]; /* Array to store tuples */ ResultRelInfo *resultRelInfo; /* ResultRelInfo for 'relid' */ BulkInsertState bistate; /* BulkInsertState for this rel */ int nused; /* number of 'slots' containing tuples */ - uint64 linenos[MAX_BUFFERED_TUPLES]; /* Line # of tuple in copy + uint64 linenos[MAX_MULTI_INSERT_TUPLES]; /* Line # of tuple in copy * stream */ } CopyMultiInsertBuffer; @@ -2351,7 +2339,7 @@ CopyMultiInsertBufferInit(ResultRelInfo *rri) CopyMultiInsertBuffer *buffer; buffer = (CopyMultiInsertBuffer *) palloc(sizeof(CopyMultiInsertBuffer)); - memset(buffer->slots, 0, sizeof(TupleTableSlot *) * MAX_BUFFERED_TUPLES); + memset(buffer->slots, 0, sizeof(TupleTableSlot *) * MAX_MULTI_INSERT_TUPLES); buffer->resultRelInfo = rri; buffer->bistate = GetBulkInsertState(); buffer->nused = 0; @@ -2410,7 +2398,7 @@ CopyMultiInsertInfoInit(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, static inline bool CopyMultiInsertInfoIsFull(CopyMultiInsertInfo *miinfo) { - if (miinfo->bufferedTuples >= MAX_BUFFERED_TUPLES || + if (miinfo->bufferedTuples >= MAX_MULTI_INSERT_TUPLES || miinfo->bufferedBytes >= MAX_BUFFERED_BYTES) return true; return false; @@ -2531,7 +2519,7 @@ CopyMultiInsertBufferCleanup(CopyMultiInsertInfo *miinfo, FreeBulkInsertState(buffer->bistate); /* Since we only create slots on demand, just drop the non-null ones. */ - for (i = 0; i < MAX_BUFFERED_TUPLES && buffer->slots[i] != NULL; i++) + for (i = 0; i < MAX_MULTI_INSERT_TUPLES && buffer->slots[i] != NULL; i++) ExecDropSingleTupleTableSlot(buffer->slots[i]); table_finish_bulk_insert(buffer->resultRelInfo->ri_RelationDesc, @@ -2620,7 +2608,7 @@ CopyMultiInsertInfoNextFreeSlot(CopyMultiInsertInfo *miinfo, int nused = buffer->nused; Assert(buffer != NULL); - Assert(nused < MAX_BUFFERED_TUPLES); + Assert(nused < MAX_MULTI_INSERT_TUPLES); if (buffer->slots[nused] == NULL) buffer->slots[nused] = table_slot_create(rri->ri_RelationDesc, NULL); diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index b7d220699f..ae3aa383f5 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -62,6 +62,10 @@ typedef struct CommandId output_cid; /* cmin to insert in output tuples */ int ti_options; /* table_tuple_insert performance options */ BulkInsertState bistate; /* bulk insert state */ + MemoryContext mi_context; /* A temporary memory context for multi insert */ + TupleTableSlot *mi_slots[MAX_MULTI_INSERT_TUPLES]; /* buffered slots for a multi insert batch. */ + int mi_slots_num; /* How many buffered slots for a multi insert batch. */ + int mi_slots_size; /* Total tuple size for a multi insert batch. */ } DR_intorel; /* utility functions for CTAS definition creation */ @@ -561,33 +565,71 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo) myState->ti_options = TABLE_INSERT_SKIP_FSM | (XLogIsNeeded() ? 0 : TABLE_INSERT_SKIP_WAL); myState->bistate = GetBulkInsertState(); + memset(myState->mi_slots, 0, sizeof(TupleTableSlot *) * MAX_MULTI_INSERT_TUPLES); + myState->mi_slots_num = 0; + myState->mi_slots_size = 0; + + /* + * Create a temporary memory context so that we can reset once per + * multi insert batch. + */ + myState->mi_context = AllocSetContextCreate(CurrentMemoryContext, + "intorel_multi_insert", + ALLOCSET_DEFAULT_SIZES); /* Not using WAL requires smgr_targblock be initially invalid */ Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber); } +static void +intorel_flush_multi_insert(DR_intorel *myState) +{ + MemoryContext oldcontext; + int i; + + oldcontext = MemoryContextSwitchTo(myState->mi_context); + + table_multi_insert(myState->rel, myState->mi_slots, + myState->mi_slots_num, myState->output_cid, + myState->ti_options, myState->bistate); + + MemoryContextReset(myState->mi_context); + MemoryContextSwitchTo(oldcontext); + + for (i = 0; i < myState->mi_slots_num; i++) + ExecClearTuple(myState->mi_slots[i]); + + myState->mi_slots_num = 0; + myState->mi_slots_size = 0; +} + /* * intorel_receive --- receive one tuple */ static bool intorel_receive(TupleTableSlot *slot, DestReceiver *self) { - DR_intorel *myState = (DR_intorel *) self; + DR_intorel *myState = (DR_intorel *) self; + TupleTableSlot *batchslot; + HeapTuple tuple; - /* - * Note that the input slot might not be of the type of the target - * relation. That's supported by table_tuple_insert(), but slightly less - * efficient than inserting with the right slot - but the alternative - * would be to copy into a slot of the right type, which would not be - * cheap either. This also doesn't allow accessing per-AM data (say a - * tuple's xmin), but since we don't do that here... - */ + if (myState->mi_slots[myState->mi_slots_num] == NULL) + { + batchslot = table_slot_create(myState->rel, NULL); + myState->mi_slots[myState->mi_slots_num] = batchslot; + } + else + batchslot = myState->mi_slots[myState->mi_slots_num]; - table_tuple_insert(myState->rel, - slot, - myState->output_cid, - myState->ti_options, - myState->bistate); + ExecCopySlot(batchslot, slot); + tuple = ExecFetchSlotHeapTuple(batchslot, true, NULL); + + myState->mi_slots_num++; + myState->mi_slots_size += tuple->t_len; + + if (myState->mi_slots_num >= MAX_MULTI_INSERT_TUPLES || + myState->mi_slots_size >= 65535) + intorel_flush_multi_insert(myState); /* We know this is a newly created relation, so there are no indexes */ @@ -601,11 +643,22 @@ static void intorel_shutdown(DestReceiver *self) { DR_intorel *myState = (DR_intorel *) self; + int i; + + if (myState->mi_slots_num != 0) + intorel_flush_multi_insert(myState); + + for (i = 0; i < MAX_MULTI_INSERT_TUPLES && myState->mi_slots[i] != NULL; i++) + ExecDropSingleTupleTableSlot(myState->mi_slots[i]); FreeBulkInsertState(myState->bistate); table_finish_bulk_insert(myState->rel, myState->ti_options); + if (myState->mi_context) + MemoryContextDelete(myState->mi_context); + myState->mi_context = NULL; + /* close rel, but keep lock until commit */ table_close(myState->rel, NoLock); myState->rel = NULL; diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 858bcb6bc9..b7c8bd42e7 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -40,6 +40,17 @@ struct TupleTableSlot; #define MaxLockTupleMode LockTupleExclusive +/* + * No more than this many tuples per MultiInsertBuffer + * + * Caution: Don't make this too big. For COPY, we could end up with this many + * CopyMultiInsertBuffer items stored in CopyMultiInsertInfo's + * multiInsertBuffers list. Increasing this can cause quadratic growth in + * memory requirements during copies into partitioned tables with a large + * number of partitions. For CTAS/MatView, the impact is similar. + */ +#define MAX_MULTI_INSERT_TUPLES 1000 + /* * Descriptor for heap table scans. */ -- 2.17.2