From d74968fe65bf0760723347b420bc2dcf8938fbb9 Mon Sep 17 00:00:00 2001 From: "Andrey M. Borodin" Date: Sun, 20 Aug 2023 23:55:31 +0300 Subject: [PATCH v22] Implement UUID v7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds function for UUID generation. Most important function here is uuidv7() which generates new UUID according to the new standard. For code readability this commit adds alias uuidv4() to function gen_random_uuid(). Also we add a function to extract timestamp from UUID v1, v6 and v7. To allow user to distinguish various UUID versions we add function uuid_extract_version(). Author: Andrey Borodin Reviewed-by: Sergey Prokhorenko, Kirk Wolak, Przemysław Sztoch Reviewed-by: Nikolay Samokhvalov, Jelte Fennema-Nio, Aleksander Alekseev Reviewed-by: Peter Eisentraut, Chris Travers, Lukas Fittl Discussion: https://postgr.es/m/CAAhFRxitJv%3DyoGnXUgeLB_O%2BM7J2BJAmb5jqAT9gZ3bij3uLDA%40mail.gmail.com --- doc/src/sgml/func.sgml | 46 +++- src/backend/utils/adt/uuid.c | 254 ++++++++++++++++++++++- src/include/catalog/pg_proc.dat | 12 ++ src/include/datatype/timestamp.h | 3 +- src/test/regress/expected/opr_sanity.out | 4 + src/test/regress/expected/uuid.out | 65 ++++++ src/test/regress/sql/uuid.sql | 25 +++ 7 files changed, 404 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 72c5175e3b..06a8f3a0d6 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -14127,13 +14127,53 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple gen_random_uuid + + uuidv4 + + + + uuidv7 + + + + uuid_extract_timestamp + + + + uuid_extract_version + + - PostgreSQL includes one function to generate a UUID: + PostgreSQL includes several functions to generate a UUID. gen_random_uuid () uuid +uuidv4 () uuid + + These functions return a version 4 (random) UUID. + +uuidv7 () uuid + + This function returns a version 7 UUID (UNIX timestamp with 1ms precision + + randomly seeded counter + random). + + + + +uuid_extract_timestamp (uuid) timestamptz + + This function extracts a timestamp with time zone from UUID versions 1, 6 and 7. + For other versions and variants this function returns NULL. The extracted timestamp + does not necessarily equate to the time of UUID generation. How close it is + to the actual time depends on the implementation that generated UUID. + The uuidv7() function provided by PostgreSQL will normally store the actual time, + with some exceptions: prevention of time leaps backwards and counter overflow + being carried to time step. + +uuid_extract_version (uuid) smallint - This function returns a version 4 (random) UUID. This is the most commonly - used type of UUID and is appropriate for most applications. + This function extracts a version bits from UUID of variant described by + IETF standard draft + (b10xx variant). For other variants this function returns NULL. diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index e9c1ec6153..80bce72276 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -13,13 +13,18 @@ #include "postgres.h" +#include + +#include "access/xlog.h" #include "common/hashfn.h" #include "lib/hyperloglog.h" #include "libpq/pqformat.h" #include "port/pg_bswap.h" -#include "utils/fmgrprotos.h" +#include "utils/builtins.h" +#include "utils/datetime.h" #include "utils/guc.h" #include "utils/sortsupport.h" +#include "utils/timestamp.h" #include "utils/uuid.h" /* sortsupport for uuid */ @@ -406,6 +411,11 @@ uuid_hash_extended(PG_FUNCTION_ARGS) return hash_any_extended(key->data, UUID_LEN, PG_GETARG_INT64(1)); } +/* + * Routine to generate UUID version 4. + * All UUID bytes are filled with strong random numbers except version and + * variant 0b10 bits. + */ Datum gen_random_uuid(PG_FUNCTION_ARGS) { @@ -425,3 +435,245 @@ gen_random_uuid(PG_FUNCTION_ARGS) PG_RETURN_UUID_P(uuid); } + +static uint32_t sequence_counter; +static uint64_t previous_timestamp = 0; + +/* + * Routine to generate UUID version 7. + * Following description is taken from RFC draft and slightly extended to + * reflect implementation specific choices. + * + * UUIDv7 Field and Bit Layout: + * ---------- + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | unix_ts_ms | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | unix_ts_ms | ver | rand_a | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |var| rand_b | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | rand_b | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * unix_ts_ms: + * 48 bit big-endian unsigned number of Unix epoch timestamp in milliseconds + * as per Section 6.1. Occupies bits 0 through 47 (octets 0-5). + * + * ver: + * The 4 bit version field as defined by Section 4.2, set to 0b0111 (7). + * Occupies bits 48 through 51 of octet 6. + * + * rand_a: + * Most significant 12 bits of 18-bit counter. This counter is designed to + * guarantee additional monotonicity as per Section 6.2 (Method 1). rand_a + * occupies bits 52 through 63 (octets 6-7). + * + * var: + * The 2 bit variant field as defined by Section 4.1, set to 0b10. Occupies + * bits 64 and 65 of octet 8. + * + * rand_b: + * Starting 6 bits are least significant 6 bits of a counter. The final 56 + * bits filled with pseudo-random data to provide uniqueness as per + * Section 6.9. rand_b Occupies bits 66 through 127 (octets 8-15). + * ---------- + * + * "Fixed-Length Dedicated Counter Bits" (Method 1) can be implemented with + * arbitrary size of a counter. We choose size 18 to reuse all space of bytes + * that are touched by ver and var fields + rand_a bytes between them. + * Whenever timestamp unix_ts_ms is moving forward, this counter bits are + * reinitialized. Reinilialization always sets most significant bit to 0, other + * bits are initialized with random numbers. This gives as approximately 192K + * UUIDs within one millisecond without overflow. This ougth to be enough for + * most practical purposes. Whenever counter overflow happens, this overflow is + * translated to increment of unix_ts_ms. So generation of UUIDs ate rate + * higher than 128MHz might lead to using timestamps ahead of time. + * + * We're not using the "Replace Left-Most Random Bits with Increased Clock + * Precision" method Section 6.2 (Method 3), because of portability concerns. + * It's unclear if all supported platforms can provide reliable microsocond + * precision time. + * + * All UUID generator state is backend-local. For UUIDs generated in one + * backend we guarantee monotonicity. UUIDs generated on different backends + * will be mostly monotonic if they are generated at frequences less than 1KHz, + * but this monotonicity is not strictly guaranteed. UUIDs generated on + * different nodes are mostly monotonic with regards to possible clock drift. + * Uniqueness of UUIDs generated at the same timestamp across different + * backends and/or nodes is guaranteed by using random bits. + */ +Datum +uuidv7(PG_FUNCTION_ARGS) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + uint64_t tms; + struct timeval tp; + bool increment_counter; + + gettimeofday(&tp, NULL); + tms = ((uint64_t) tp.tv_sec) * 1000 + (tp.tv_usec) / 1000; + /* time from clock is protected from backward leaps */ + increment_counter = (tms <= previous_timestamp); + + if (increment_counter) + { + /* + * Time did not advance from the previous generation, we must + * increment counter + */ + ++sequence_counter; + if (sequence_counter > 0x3ffff) + { + /* We only have 18-bit counter */ + sequence_counter = 0; + previous_timestamp++; + } + + /* protection from leap backward */ + tms = previous_timestamp; + + /* fill everything after the timestamp and counter with random bytes */ + if (!pg_strong_random(&uuid->data[8], UUID_LEN - 8)) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate random values"))); + + /* most significant 4 bits of 18-bit counter */ + uuid->data[6] = (unsigned char) (sequence_counter >> 14); + /* next 8 bits */ + uuid->data[7] = (unsigned char) (sequence_counter >> 6); + /* least significant 6 bits */ + uuid->data[8] = (unsigned char) (sequence_counter); + } + else + { + /* fill everything after the timestamp with random bytes */ + if (!pg_strong_random(&uuid->data[6], UUID_LEN - 6)) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate random values"))); + + /* + * Left-most counter bits are initialized as zero for the sole purpose + * of guarding against counter rollovers. See section "Fixed-Length + * Dedicated Counter Seeding" + * https://datatracker.ietf.org/doc/html/draft-ietf-uuidrev-rfc4122bis#monotonicity_counters + */ + uuid->data[6] = (uuid->data[6] & 0xf7); + + /* read randomly initialized bits of counter */ + sequence_counter = ((uint32_t) uuid->data[8] & 0x3f) + + (((uint32_t) uuid->data[7]) << 6) + + (((uint32_t) uuid->data[6] & 0x0f) << 14); + + previous_timestamp = tms; + } + + /* Fill in time part */ + uuid->data[0] = (unsigned char) (tms >> 40); + uuid->data[1] = (unsigned char) (tms >> 32); + uuid->data[2] = (unsigned char) (tms >> 24); + uuid->data[3] = (unsigned char) (tms >> 16); + uuid->data[4] = (unsigned char) (tms >> 8); + uuid->data[5] = (unsigned char) tms; + + /* + * Set magic numbers for a "version 7" (pseudorandom) UUID, see + * https://datatracker.ietf.org/doc/html/draft-ietf-uuidrev-rfc4122bis + */ + /* set version field, top four bits are 0, 1, 1, 1 */ + uuid->data[6] = (uuid->data[6] & 0x0f) | 0x70; + /* set variant field, top two bits are 1, 0 */ + uuid->data[8] = (uuid->data[8] & 0x3f) | 0x80; + + PG_RETURN_UUID_P(uuid); +} + +/* + * Routine to extract UUID version from variant 0b10 + * Returns NULL if UUID is not 0b10 or version is not 1,6 or7. + */ +Datum +uuid_extract_timestamp(PG_FUNCTION_ARGS) +{ + pg_uuid_t *uuid = PG_GETARG_UUID_P(0); + TimestampTz ts; + uint64_t tms; + + if ((uuid->data[8] & 0xc0) != 0x80) + PG_RETURN_NULL(); + + if ((uuid->data[6] & 0xf0) == 0x70) + { + tms = uuid->data[5]; + tms += ((uint64_t) uuid->data[4]) << 8; + tms += ((uint64_t) uuid->data[3]) << 16; + tms += ((uint64_t) uuid->data[2]) << 24; + tms += ((uint64_t) uuid->data[1]) << 32; + tms += ((uint64_t) uuid->data[0]) << 40; + + /* convert ms to us, then adjust */ + ts = (TimestampTz) (tms * 1000) - + (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY * USECS_PER_SEC; + + PG_RETURN_TIMESTAMPTZ(ts); + } + + if ((uuid->data[6] & 0xf0) == 0x10) + { + tms = ((uint64_t) uuid->data[0]) << 24; + tms += ((uint64_t) uuid->data[1]) << 16; + tms += ((uint64_t) uuid->data[2]) << 8; + tms += ((uint64_t) uuid->data[3]); + tms += ((uint64_t) uuid->data[4]) << 40; + tms += ((uint64_t) uuid->data[5]) << 32; + tms += (((uint64_t) uuid->data[6]) & 0xf) << 56; + tms += ((uint64_t) uuid->data[7]) << 48; + + /* convert 100-ns intervals to us, then adjust */ + ts = (TimestampTz) (tms / 10) - + ((uint64_t) POSTGRES_EPOCH_JDATE - GREGORIAN_EPOCH_JDATE) * SECS_PER_DAY * USECS_PER_SEC; + + PG_RETURN_TIMESTAMPTZ(ts); + } + + if ((uuid->data[6] & 0xf0) == 0x60) + { + tms = ((uint64_t) uuid->data[0]) << 52; + tms += ((uint64_t) uuid->data[1]) << 44; + tms += ((uint64_t) uuid->data[2]) << 36; + tms += ((uint64_t) uuid->data[3]) << 28; + tms += ((uint64_t) uuid->data[4]) << 20; + tms += ((uint64_t) uuid->data[5]) << 12; + tms += (((uint64_t) uuid->data[6]) & 0xf) << 8; + tms += ((uint64_t) uuid->data[7]); + + /* convert 100-ns intervals to us, then adjust */ + ts = (TimestampTz) (tms / 10) - + ((uint64_t) POSTGRES_EPOCH_JDATE - GREGORIAN_EPOCH_JDATE) * SECS_PER_DAY * USECS_PER_SEC; + + PG_RETURN_TIMESTAMPTZ(ts); + } + + PG_RETURN_NULL(); +} + +/* + * Routine to extract UUID version from variant 0b10 + * Returns NULL if UUID is not 0b10 + */ +Datum +uuid_extract_version(PG_FUNCTION_ARGS) +{ + pg_uuid_t *uuid = PG_GETARG_UUID_P(0); + uint16_t result; + + if ((uuid->data[8] & 0xc0) != 0x80) + PG_RETURN_NULL(); + result = uuid->data[6] >> 4; + + PG_RETURN_UINT16(result); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 700f7daf7b..ddac2f51f6 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -9170,6 +9170,18 @@ { oid => '3432', descr => 'generate random UUID', proname => 'gen_random_uuid', proleakproof => 't', provolatile => 'v', prorettype => 'uuid', proargtypes => '', prosrc => 'gen_random_uuid' }, +{ oid => '9895', descr => 'generate UUID version 4', + proname => 'uuidv4', proleakproof => 't', provolatile => 'v', + prorettype => 'uuid', proargtypes => '', prosrc => 'gen_random_uuid' }, +{ oid => '9896', descr => 'generate UUID version 7', + proname => 'uuidv7', proleakproof => 't', provolatile => 'v', + prorettype => 'uuid', proargtypes => '', prosrc => 'uuidv7' }, +{ oid => '9897', descr => 'extract timestamp from UUID version 1, 6 or 7', + proname => 'uuid_extract_timestamp', proleakproof => 't', + prorettype => 'timestamptz', proargtypes => 'uuid', prosrc => 'uuid_extract_timestamp' }, +{ oid => '9898', descr => 'extract version from RFC 4122 UUID', + proname => 'uuid_extract_version', proleakproof => 't', + prorettype => 'int2', proargtypes => 'uuid', prosrc => 'uuid_extract_version' }, # pg_lsn { oid => '3229', descr => 'I/O', diff --git a/src/include/datatype/timestamp.h b/src/include/datatype/timestamp.h index 3a37cb661e..652aeb428e 100644 --- a/src/include/datatype/timestamp.h +++ b/src/include/datatype/timestamp.h @@ -230,9 +230,10 @@ struct pg_itm_in ((y) < JULIAN_MAXYEAR || \ ((y) == JULIAN_MAXYEAR && ((m) < JULIAN_MAXMONTH)))) -/* Julian-date equivalents of Day 0 in Unix and Postgres reckoning */ +/* Julian-date equivalents of Day 0 in Unix, Postgres and Gregorian epochs */ #define UNIX_EPOCH_JDATE 2440588 /* == date2j(1970, 1, 1) */ #define POSTGRES_EPOCH_JDATE 2451545 /* == date2j(2000, 1, 1) */ +#define GREGORIAN_EPOCH_JDATE 2299161 /* == date2j(1582,10,15) */ /* * Range limits for dates and timestamps. diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 7610b011d6..2b5d67b7f0 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -872,6 +872,10 @@ xid8ge(xid8,xid8) xid8eq(xid8,xid8) xid8ne(xid8,xid8) xid8cmp(xid8,xid8) +uuidv4() +uuidv7() +uuid_extract_timestamp(uuid) +uuid_extract_version(uuid) -- restore normal output mode \a\t -- List of functions used by libpq's fe-lobj.c diff --git a/src/test/regress/expected/uuid.out b/src/test/regress/expected/uuid.out index 8e7f21910d..15dab4e1c2 100644 --- a/src/test/regress/expected/uuid.out +++ b/src/test/regress/expected/uuid.out @@ -168,5 +168,70 @@ SELECT count(DISTINCT guid_field) FROM guid1; 2 (1 row) +-- test of uuidv4() alias +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (uuidv4()); +INSERT INTO guid1 (guid_field) VALUES (uuidv4()); +SELECT count(DISTINCT guid_field) FROM guid1; + count +------- + 2 +(1 row) + +-- generation test for v7 +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (uuidv7()); +INSERT INTO guid1 (guid_field) VALUES (uuidv7()); +SELECT count(DISTINCT guid_field) FROM guid1; + count +------- + 2 +(1 row) + +-- support functions for UUID versions and variants +SELECT uuid_extract_version(uuidv7()); + uuid_extract_version +---------------------- + 7 +(1 row) + +SELECT uuid_extract_version('11111111-1111-1111-1111-111111111111') IS NULL; + ?column? +---------- + t +(1 row) + +SELECT uuid_extract_version('11111111-1111-5111-8111-111111111111'); + uuid_extract_version +---------------------- + 5 +(1 row) + +-- uuid_extract_timestamp() must refuse to accept non-UUIDv7 +SELECT uuid_extract_timestamp(gen_random_uuid()); + uuid_extract_timestamp +------------------------ + +(1 row) + +-- extract UUID v1, v6 and v7 timestamp +SELECT uuid_extract_timestamp('C232AB00-9414-11EC-B3C8-9F6BDECED846') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; + ?column? +---------- + t +(1 row) + +SELECT uuid_extract_timestamp('1EC9414C-232A-6B00-B3C8-9F6BDECED846') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; + ?column? +---------- + t +(1 row) + +SELECT uuid_extract_timestamp('017F22E2-79B0-7CC3-98C4-DC0C0C07398F') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; + ?column? +---------- + t +(1 row) + -- clean up DROP TABLE guid1, guid2 CASCADE; diff --git a/src/test/regress/sql/uuid.sql b/src/test/regress/sql/uuid.sql index 9a8f437c7d..0d1b041b6c 100644 --- a/src/test/regress/sql/uuid.sql +++ b/src/test/regress/sql/uuid.sql @@ -85,5 +85,30 @@ INSERT INTO guid1 (guid_field) VALUES (gen_random_uuid()); INSERT INTO guid1 (guid_field) VALUES (gen_random_uuid()); SELECT count(DISTINCT guid_field) FROM guid1; +-- test of uuidv4() alias +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (uuidv4()); +INSERT INTO guid1 (guid_field) VALUES (uuidv4()); +SELECT count(DISTINCT guid_field) FROM guid1; + +-- generation test for v7 +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (uuidv7()); +INSERT INTO guid1 (guid_field) VALUES (uuidv7()); +SELECT count(DISTINCT guid_field) FROM guid1; + +-- support functions for UUID versions and variants +SELECT uuid_extract_version(uuidv7()); +SELECT uuid_extract_version('11111111-1111-1111-1111-111111111111') IS NULL; +SELECT uuid_extract_version('11111111-1111-5111-8111-111111111111'); + +-- uuid_extract_timestamp() must refuse to accept non-UUIDv7 +SELECT uuid_extract_timestamp(gen_random_uuid()); + +-- extract UUID v1, v6 and v7 timestamp +SELECT uuid_extract_timestamp('C232AB00-9414-11EC-B3C8-9F6BDECED846') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; +SELECT uuid_extract_timestamp('1EC9414C-232A-6B00-B3C8-9F6BDECED846') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; +SELECT uuid_extract_timestamp('017F22E2-79B0-7CC3-98C4-DC0C0C07398F') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; + -- clean up DROP TABLE guid1, guid2 CASCADE; -- 2.37.1 (Apple Git-137.1)