kevinjqliu commented on code in PR #1469:
URL: https://github.com/apache/iceberg-python/pull/1469#discussion_r1902181377
##########
tests/integration/test_rest_catalog.py:
##########
@@ -16,34 +16,788 @@
# under the License.
# pylint:disable=redefined-outer-name
+
+from typing import Any, Dict
+
import pytest
+from pyiceberg.catalog import PropertiesUpdateSummary
from pyiceberg.catalog.rest import RestCatalog
+from pyiceberg.exceptions import (
+ BadRequestError,
+ NamespaceAlreadyExistsError,
+ NoSuchIdentifierError,
+ NoSuchNamespaceError,
+ NoSuchTableError,
+ NoSuchViewError,
+ OAuthError,
+ TableAlreadyExistsError,
+)
+from pyiceberg.io import load_file_io
+from pyiceberg.partitioning import PartitionField, PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.table import Table
+from pyiceberg.table.metadata import TableMetadataV2
+from pyiceberg.table.sorting import NullOrder, SortDirection, SortField,
SortOrder
+from pyiceberg.transforms import IdentityTransform, TruncateTransform
+
+TEST_NAMESPACE_IDENTIFIER = ("rest_integration_ns",)
+TEST_TABLE_IDENTIFIER = ("rest_integration_ns", "rest_integration_tbl")
+TEST_TABLE_IDENTIFIER_RENAME = ("rest_integration_ns",
"renamed_rest_integration_tbl")
+TEST_URI = "http://localhost:8181";
+TEST_CREDENTIALS = "client:secret"
+
+EXAMPLE_table_metadata_no_snapshot_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [{"spec-id": 0, "fields": ()}],
+ "default-spec-id": 0,
+ "last-partition-id": 999,
+ "default-sort-order-id": 0,
+ "sort-orders": [{"order-id": 0, "fields": []}],
+ "properties": {
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+EXAMPLE_table_metadata_no_snapshot_partitioned_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [
+ {"spec-id": 0, "fields": ({"source-id": 1, "field-id": 1000,
"transform": "truncate[3]", "name": "id"},)}
+ ],
+ "default-spec-id": 0,
+ "last-partition-id": 1000,
+ "default-sort-order-id": 1,
+ "sort-orders": [
+ {
+ "order-id": 1,
+ "fields": [
+ {"source-id": 2, "transform": "identity", "direction":
SortDirection.ASC, "null-order": NullOrder.NULLS_FIRST}
+ ],
+ }
+ ],
+ "properties": {
+ "owner": "fokko",
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_v2
+
-TEST_NAMESPACE_IDENTIFIER = "TEST NS"
+@pytest.fixture
+def table_metadata_no_snapshot_partitioned_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_partitioned_v2
+
+
+@pytest.fixture
+def
rest_integration_example_metadata_partitioned_v2(table_metadata_no_snapshot_partitioned_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_partitioned_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def example_table_metadata_with_no_location(table_metadata_no_snapshot_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_rest(table_metadata_no_snapshot_v2: Dict[str,
Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
@pytest.mark.integration
-@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")])
-def test_namespace_exists(catalog: RestCatalog) -> None:
- if not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER):
+@pytest.fixture(scope="function")
+@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up")])
Review Comment:
nit: you can use something like this to automatically run clean up after
every test
https://stackoverflow.com/questions/22627659/run-code-before-and-after-each-test-in-py-test
```
@pytest.fixture(autouse=True)
def cleanup():
# Code that will run before your test, for example:
yield
# Code that will run after your test, for example:
```
##########
tests/integration/test_rest_catalog.py:
##########
@@ -16,34 +16,788 @@
# under the License.
# pylint:disable=redefined-outer-name
+
+from typing import Any, Dict
+
import pytest
+from pyiceberg.catalog import PropertiesUpdateSummary
from pyiceberg.catalog.rest import RestCatalog
+from pyiceberg.exceptions import (
+ BadRequestError,
+ NamespaceAlreadyExistsError,
+ NoSuchIdentifierError,
+ NoSuchNamespaceError,
+ NoSuchTableError,
+ NoSuchViewError,
+ OAuthError,
+ TableAlreadyExistsError,
+)
+from pyiceberg.io import load_file_io
+from pyiceberg.partitioning import PartitionField, PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.table import Table
+from pyiceberg.table.metadata import TableMetadataV2
+from pyiceberg.table.sorting import NullOrder, SortDirection, SortField,
SortOrder
+from pyiceberg.transforms import IdentityTransform, TruncateTransform
+
+TEST_NAMESPACE_IDENTIFIER = ("rest_integration_ns",)
+TEST_TABLE_IDENTIFIER = ("rest_integration_ns", "rest_integration_tbl")
+TEST_TABLE_IDENTIFIER_RENAME = ("rest_integration_ns",
"renamed_rest_integration_tbl")
+TEST_URI = "http://localhost:8181";
+TEST_CREDENTIALS = "client:secret"
+
+EXAMPLE_table_metadata_no_snapshot_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [{"spec-id": 0, "fields": ()}],
+ "default-spec-id": 0,
+ "last-partition-id": 999,
+ "default-sort-order-id": 0,
+ "sort-orders": [{"order-id": 0, "fields": []}],
+ "properties": {
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+EXAMPLE_table_metadata_no_snapshot_partitioned_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [
+ {"spec-id": 0, "fields": ({"source-id": 1, "field-id": 1000,
"transform": "truncate[3]", "name": "id"},)}
+ ],
+ "default-spec-id": 0,
+ "last-partition-id": 1000,
+ "default-sort-order-id": 1,
+ "sort-orders": [
+ {
+ "order-id": 1,
+ "fields": [
+ {"source-id": 2, "transform": "identity", "direction":
SortDirection.ASC, "null-order": NullOrder.NULLS_FIRST}
+ ],
+ }
+ ],
+ "properties": {
+ "owner": "fokko",
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_v2
+
-TEST_NAMESPACE_IDENTIFIER = "TEST NS"
+@pytest.fixture
+def table_metadata_no_snapshot_partitioned_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_partitioned_v2
+
+
+@pytest.fixture
+def
rest_integration_example_metadata_partitioned_v2(table_metadata_no_snapshot_partitioned_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_partitioned_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def example_table_metadata_with_no_location(table_metadata_no_snapshot_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_rest(table_metadata_no_snapshot_v2: Dict[str,
Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
@pytest.mark.integration
-@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")])
-def test_namespace_exists(catalog: RestCatalog) -> None:
- if not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER):
+@pytest.fixture(scope="function")
+@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up")])
+def test_clean_up(catalog: RestCatalog) -> None:
+ for namespaces_tuple in catalog.list_namespaces():
+ namespace_name = namespaces_tuple[0]
+ if TEST_NAMESPACE_IDENTIFIER[0] in namespace_name:
+ for identifier in catalog.list_tables(namespace_name):
+ catalog.purge_table(identifier)
+ if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER):
+ catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER)
Review Comment:
since we only have 2 tables and 1 namespace:
```
TEST_NAMESPACE_IDENTIFIER = ("rest_integration_ns",)
TEST_TABLE_IDENTIFIER = ("rest_integration_ns", "rest_integration_tbl")
TEST_TABLE_IDENTIFIER_RENAME = ("rest_integration_ns",
"renamed_rest_integration_tbl")
```
we can drop all tables in the `TEST_NAMESPACE_IDENTIFIER` namespace, and
then remove the remove
##########
tests/integration/test_rest_catalog.py:
##########
@@ -16,34 +16,788 @@
# under the License.
# pylint:disable=redefined-outer-name
+
+from typing import Any, Dict
+
import pytest
+from pyiceberg.catalog import PropertiesUpdateSummary
from pyiceberg.catalog.rest import RestCatalog
+from pyiceberg.exceptions import (
+ BadRequestError,
+ NamespaceAlreadyExistsError,
+ NoSuchIdentifierError,
+ NoSuchNamespaceError,
+ NoSuchTableError,
+ NoSuchViewError,
+ OAuthError,
+ TableAlreadyExistsError,
+)
+from pyiceberg.io import load_file_io
+from pyiceberg.partitioning import PartitionField, PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.table import Table
+from pyiceberg.table.metadata import TableMetadataV2
+from pyiceberg.table.sorting import NullOrder, SortDirection, SortField,
SortOrder
+from pyiceberg.transforms import IdentityTransform, TruncateTransform
+
+TEST_NAMESPACE_IDENTIFIER = ("rest_integration_ns",)
+TEST_TABLE_IDENTIFIER = ("rest_integration_ns", "rest_integration_tbl")
+TEST_TABLE_IDENTIFIER_RENAME = ("rest_integration_ns",
"renamed_rest_integration_tbl")
+TEST_URI = "http://localhost:8181";
+TEST_CREDENTIALS = "client:secret"
+
+EXAMPLE_table_metadata_no_snapshot_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [{"spec-id": 0, "fields": ()}],
+ "default-spec-id": 0,
+ "last-partition-id": 999,
+ "default-sort-order-id": 0,
+ "sort-orders": [{"order-id": 0, "fields": []}],
+ "properties": {
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+EXAMPLE_table_metadata_no_snapshot_partitioned_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [
+ {"spec-id": 0, "fields": ({"source-id": 1, "field-id": 1000,
"transform": "truncate[3]", "name": "id"},)}
+ ],
+ "default-spec-id": 0,
+ "last-partition-id": 1000,
+ "default-sort-order-id": 1,
+ "sort-orders": [
+ {
+ "order-id": 1,
+ "fields": [
+ {"source-id": 2, "transform": "identity", "direction":
SortDirection.ASC, "null-order": NullOrder.NULLS_FIRST}
+ ],
+ }
+ ],
+ "properties": {
+ "owner": "fokko",
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_v2
+
-TEST_NAMESPACE_IDENTIFIER = "TEST NS"
+@pytest.fixture
+def table_metadata_no_snapshot_partitioned_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_partitioned_v2
+
+
+@pytest.fixture
+def
rest_integration_example_metadata_partitioned_v2(table_metadata_no_snapshot_partitioned_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_partitioned_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def example_table_metadata_with_no_location(table_metadata_no_snapshot_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_rest(table_metadata_no_snapshot_v2: Dict[str,
Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
@pytest.mark.integration
-@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")])
-def test_namespace_exists(catalog: RestCatalog) -> None:
- if not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER):
+@pytest.fixture(scope="function")
+@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up")])
+def test_clean_up(catalog: RestCatalog) -> None:
+ for namespaces_tuple in catalog.list_namespaces():
+ namespace_name = namespaces_tuple[0]
+ if TEST_NAMESPACE_IDENTIFIER[0] in namespace_name:
+ for identifier in catalog.list_tables(namespace_name):
+ catalog.purge_table(identifier)
+ if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER):
+ catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER)
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("catalog,clean_up",
[(pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up"))])
+def test_create_namespace_200(catalog: RestCatalog, clean_up: Any) -> None:
Review Comment:
nit: the naming is confusing since we dont assert the 200 status code.
how about just `test_create_namespace`
##########
tests/integration/test_rest_catalog.py:
##########
@@ -16,34 +16,788 @@
# under the License.
# pylint:disable=redefined-outer-name
+
+from typing import Any, Dict
+
import pytest
+from pyiceberg.catalog import PropertiesUpdateSummary
from pyiceberg.catalog.rest import RestCatalog
+from pyiceberg.exceptions import (
+ BadRequestError,
+ NamespaceAlreadyExistsError,
+ NoSuchIdentifierError,
+ NoSuchNamespaceError,
+ NoSuchTableError,
+ NoSuchViewError,
+ OAuthError,
+ TableAlreadyExistsError,
+)
+from pyiceberg.io import load_file_io
+from pyiceberg.partitioning import PartitionField, PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.table import Table
+from pyiceberg.table.metadata import TableMetadataV2
+from pyiceberg.table.sorting import NullOrder, SortDirection, SortField,
SortOrder
+from pyiceberg.transforms import IdentityTransform, TruncateTransform
+
+TEST_NAMESPACE_IDENTIFIER = ("rest_integration_ns",)
+TEST_TABLE_IDENTIFIER = ("rest_integration_ns", "rest_integration_tbl")
+TEST_TABLE_IDENTIFIER_RENAME = ("rest_integration_ns",
"renamed_rest_integration_tbl")
+TEST_URI = "http://localhost:8181";
+TEST_CREDENTIALS = "client:secret"
+
+EXAMPLE_table_metadata_no_snapshot_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [{"spec-id": 0, "fields": ()}],
+ "default-spec-id": 0,
+ "last-partition-id": 999,
+ "default-sort-order-id": 0,
+ "sort-orders": [{"order-id": 0, "fields": []}],
+ "properties": {
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+EXAMPLE_table_metadata_no_snapshot_partitioned_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [
+ {"spec-id": 0, "fields": ({"source-id": 1, "field-id": 1000,
"transform": "truncate[3]", "name": "id"},)}
+ ],
+ "default-spec-id": 0,
+ "last-partition-id": 1000,
+ "default-sort-order-id": 1,
+ "sort-orders": [
+ {
+ "order-id": 1,
+ "fields": [
+ {"source-id": 2, "transform": "identity", "direction":
SortDirection.ASC, "null-order": NullOrder.NULLS_FIRST}
+ ],
+ }
+ ],
+ "properties": {
+ "owner": "fokko",
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_v2
+
-TEST_NAMESPACE_IDENTIFIER = "TEST NS"
+@pytest.fixture
+def table_metadata_no_snapshot_partitioned_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_partitioned_v2
+
+
+@pytest.fixture
+def
rest_integration_example_metadata_partitioned_v2(table_metadata_no_snapshot_partitioned_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_partitioned_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def example_table_metadata_with_no_location(table_metadata_no_snapshot_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_rest(table_metadata_no_snapshot_v2: Dict[str,
Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
@pytest.mark.integration
-@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")])
-def test_namespace_exists(catalog: RestCatalog) -> None:
- if not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER):
+@pytest.fixture(scope="function")
+@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up")])
+def test_clean_up(catalog: RestCatalog) -> None:
+ for namespaces_tuple in catalog.list_namespaces():
+ namespace_name = namespaces_tuple[0]
+ if TEST_NAMESPACE_IDENTIFIER[0] in namespace_name:
+ for identifier in catalog.list_tables(namespace_name):
+ catalog.purge_table(identifier)
+ if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER):
+ catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER)
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("catalog,clean_up",
[(pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up"))])
+def test_create_namespace_200(catalog: RestCatalog, clean_up: Any) -> None:
+ catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER)
+ assert TEST_NAMESPACE_IDENTIFIER in catalog.list_namespaces()
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("catalog,clean_up",
[(pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up"))])
+def test_create_namespace_if_exists_409(catalog: RestCatalog, clean_up: Any)
-> None:
+ catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER)
+ catalog.create_namespace_if_not_exists(TEST_NAMESPACE_IDENTIFIER)
+ assert TEST_NAMESPACE_IDENTIFIER in catalog.list_namespaces()
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("catalog,clean_up",
[(pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up"))])
+def test_list_namespaces_200(catalog: RestCatalog, clean_up: Any) -> None:
+ catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER)
+ assert catalog.list_namespaces() == [("default",),
TEST_NAMESPACE_IDENTIFIER]
Review Comment:
nit: `TEST_NAMESPACE_IDENTIFIER in catalog.list_namespaces()` so we dont
care about the other namespace
##########
tests/integration/test_rest_catalog.py:
##########
@@ -16,34 +16,788 @@
# under the License.
# pylint:disable=redefined-outer-name
+
+from typing import Any, Dict
+
import pytest
+from pyiceberg.catalog import PropertiesUpdateSummary
from pyiceberg.catalog.rest import RestCatalog
+from pyiceberg.exceptions import (
+ BadRequestError,
+ NamespaceAlreadyExistsError,
+ NoSuchIdentifierError,
+ NoSuchNamespaceError,
+ NoSuchTableError,
+ NoSuchViewError,
+ OAuthError,
+ TableAlreadyExistsError,
+)
+from pyiceberg.io import load_file_io
+from pyiceberg.partitioning import PartitionField, PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.table import Table
+from pyiceberg.table.metadata import TableMetadataV2
+from pyiceberg.table.sorting import NullOrder, SortDirection, SortField,
SortOrder
+from pyiceberg.transforms import IdentityTransform, TruncateTransform
+
+TEST_NAMESPACE_IDENTIFIER = ("rest_integration_ns",)
+TEST_TABLE_IDENTIFIER = ("rest_integration_ns", "rest_integration_tbl")
+TEST_TABLE_IDENTIFIER_RENAME = ("rest_integration_ns",
"renamed_rest_integration_tbl")
+TEST_URI = "http://localhost:8181";
+TEST_CREDENTIALS = "client:secret"
+
+EXAMPLE_table_metadata_no_snapshot_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [{"spec-id": 0, "fields": ()}],
+ "default-spec-id": 0,
+ "last-partition-id": 999,
+ "default-sort-order-id": 0,
+ "sort-orders": [{"order-id": 0, "fields": []}],
+ "properties": {
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+EXAMPLE_table_metadata_no_snapshot_partitioned_v2 = {
+ "format-version": 2,
+ "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29",
+ "location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "last-updated-ms": 1657810967051,
+ "last-column-id": 3,
+ "schema": {
+ "type": "struct",
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ "fields": [
+ {"id": 1, "name": "foo", "required": False, "type": "string"},
+ {"id": 2, "name": "bar", "required": True, "type": "int"},
+ {"id": 3, "name": "baz", "required": False, "type": "boolean"},
+ ],
+ },
+ "current-schema-id": 0,
+ "schemas": [
+ {
+ "type": "struct",
+ "fields": (
+ {"id": 1, "name": "foo", "type": "string", "required": False},
+ {"id": 2, "name": "bar", "type": "int", "required": True},
+ {"id": 3, "name": "baz", "type": "boolean", "required": False},
+ ),
+ "schema-id": 0,
+ "identifier-field-ids": [2],
+ }
+ ],
+ "partition-specs": [
+ {"spec-id": 0, "fields": ({"source-id": 1, "field-id": 1000,
"transform": "truncate[3]", "name": "id"},)}
+ ],
+ "default-spec-id": 0,
+ "last-partition-id": 1000,
+ "default-sort-order-id": 1,
+ "sort-orders": [
+ {
+ "order-id": 1,
+ "fields": [
+ {"source-id": 2, "transform": "identity", "direction":
SortDirection.ASC, "null-order": NullOrder.NULLS_FIRST}
+ ],
+ }
+ ],
+ "properties": {
+ "owner": "fokko",
+ "write.parquet.compression-codec": "zstd",
+ },
+ "refs": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+}
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_v2
+
-TEST_NAMESPACE_IDENTIFIER = "TEST NS"
+@pytest.fixture
+def table_metadata_no_snapshot_partitioned_v2() -> Dict[str, Any]:
+ return EXAMPLE_table_metadata_no_snapshot_partitioned_v2
+
+
+@pytest.fixture
+def
rest_integration_example_metadata_partitioned_v2(table_metadata_no_snapshot_partitioned_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_partitioned_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def example_table_metadata_with_no_location(table_metadata_no_snapshot_v2:
Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
+
+
+@pytest.fixture
+def table_metadata_no_snapshot_rest(table_metadata_no_snapshot_v2: Dict[str,
Any]) -> Dict[str, Any]:
+ return {
+ "metadata-location":
f"s3://warehouse/{TEST_TABLE_IDENTIFIER[0]}/{TEST_TABLE_IDENTIFIER[1]}",
+ "metadata": table_metadata_no_snapshot_v2,
+ "config": {
+ "client.factory":
"io.tabular.iceberg.catalog.TabularAwsClientFactory",
+ "region": "us-west-2",
+ },
+ }
@pytest.mark.integration
-@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")])
-def test_namespace_exists(catalog: RestCatalog) -> None:
- if not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER):
+@pytest.fixture(scope="function")
+@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up")])
+def test_clean_up(catalog: RestCatalog) -> None:
+ for namespaces_tuple in catalog.list_namespaces():
+ namespace_name = namespaces_tuple[0]
+ if TEST_NAMESPACE_IDENTIFIER[0] in namespace_name:
+ for identifier in catalog.list_tables(namespace_name):
+ catalog.purge_table(identifier)
+ if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER):
+ catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER)
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("catalog,clean_up",
[(pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up"))])
+def test_create_namespace_200(catalog: RestCatalog, clean_up: Any) -> None:
+ catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER)
+ assert TEST_NAMESPACE_IDENTIFIER in catalog.list_namespaces()
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("catalog,clean_up",
[(pytest.lazy_fixture("session_catalog"),
pytest.lazy_fixture("test_clean_up"))])
+def test_create_namespace_if_exists_409(catalog: RestCatalog, clean_up: Any)
-> None:
+ catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER)
+ catalog.create_namespace_if_not_exists(TEST_NAMESPACE_IDENTIFIER)
+ assert TEST_NAMESPACE_IDENTIFIER in catalog.list_namespaces()
Review Comment:
nit: similar to above with naming, the test name can be more descriptive. i
think here we're testing that `create_namespace_if_not_exists` does not throw
`NamespaceAlreadyExistsError` when the namespace already exists
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
