Commit 34477be
authored
test: check extreme DATE/DATETIME values can be loaded from pandas DataFrame (#1078)
Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly:
- [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea
- [ ] Ensure the tests and linter pass
- [ ] Code coverage does not decrease (if any source code was changed)
- [ ] Appropriate docs were updated (if necessary)
Towards #1076 🦕
(edit: moved to googleapis/python-db-dtypes-pandas#45 )1 parent e73f1b4 commit 34477be
1 file changed
Lines changed: 20 additions & 21 deletions
Lines changed: 20 additions & 21 deletions
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -268,7 +268,7 @@ def test_load_table_from_dataframe_w_nulls(bigquery_client, dataset_id): | |||
| 268 | 268 | See: https://github.com/googleapis/google-cloud-python/issues/7370 | |
| 269 | 269 | """ | |
| 270 | 270 | # Schema with all scalar types. | |
| 271 | - scalars_schema = ( | ||
| 271 | + table_schema = ( | ||
| 272 | 272 | bigquery.SchemaField("bool_col", "BOOLEAN"), | |
| 273 | 273 | bigquery.SchemaField("bytes_col", "BYTES"), | |
| 274 | 274 | bigquery.SchemaField("date_col", "DATE"), | |
@@ -283,15 +283,6 @@ def test_load_table_from_dataframe_w_nulls(bigquery_client, dataset_id): | |||
| 283 | 283 | bigquery.SchemaField("ts_col", "TIMESTAMP"), | |
| 284 | 284 | ) | |
| 285 | 285 | ||
| 286 | - table_schema = scalars_schema + ( | ||
| 287 | - # TODO: Array columns can't be read due to NULLABLE versus REPEATED | ||
| 288 | - # mode mismatch. See: | ||
| 289 | - # https://issuetracker.google.com/133415569#comment3 | ||
| 290 | - # bigquery.SchemaField("array_col", "INTEGER", mode="REPEATED"), | ||
| 291 | - # TODO: Support writing StructArrays to Parquet. See: | ||
| 292 | - # https://jira.apache.org/jira/browse/ARROW-2587 | ||
| 293 | - # bigquery.SchemaField("struct_col", "RECORD", fields=scalars_schema), | ||
| 294 | - ) | ||
| 295 | 286 | num_rows = 100 | |
| 296 | 287 | nulls = [None] * num_rows | |
| 297 | 288 | df_data = [ | |
@@ -372,7 +363,8 @@ def test_load_table_from_dataframe_w_explicit_schema(bigquery_client, dataset_id | |||
| 372 | 363 | # See: | |
| 373 | 364 | # https://github.com/googleapis/python-bigquery/issues/61 | |
| 374 | 365 | # https://issuetracker.google.com/issues/151765076 | |
| 375 | - scalars_schema = ( | ||
| 366 | + table_schema = ( | ||
| 367 | + bigquery.SchemaField("row_num", "INTEGER"), | ||
| 376 | 368 | bigquery.SchemaField("bool_col", "BOOLEAN"), | |
| 377 | 369 | bigquery.SchemaField("bytes_col", "BYTES"), | |
| 378 | 370 | bigquery.SchemaField("date_col", "DATE"), | |
@@ -387,17 +379,8 @@ def test_load_table_from_dataframe_w_explicit_schema(bigquery_client, dataset_id | |||
| 387 | 379 | bigquery.SchemaField("ts_col", "TIMESTAMP"), | |
| 388 | 380 | ) | |
| 389 | 381 | ||
| 390 | - table_schema = scalars_schema + ( | ||
| 391 | - # TODO: Array columns can't be read due to NULLABLE versus REPEATED | ||
| 392 | - # mode mismatch. See: | ||
| 393 | - # https://issuetracker.google.com/133415569#comment3 | ||
| 394 | - # bigquery.SchemaField("array_col", "INTEGER", mode="REPEATED"), | ||
| 395 | - # TODO: Support writing StructArrays to Parquet. See: | ||
| 396 | - # https://jira.apache.org/jira/browse/ARROW-2587 | ||
| 397 | - # bigquery.SchemaField("struct_col", "RECORD", fields=scalars_schema), | ||
| 398 | - ) | ||
| 399 | - | ||
| 400 | 382 | df_data = [ | |
| 383 | + ("row_num", [1, 2, 3]), | ||
| 401 | 384 | ("bool_col", [True, None, False]), | |
| 402 | 385 | ("bytes_col", [b"abc", None, b"def"]), | |
| 403 | 386 | ("date_col", [datetime.date(1, 1, 1), None, datetime.date(9999, 12, 31)]), | |
@@ -464,6 +447,22 @@ def test_load_table_from_dataframe_w_explicit_schema(bigquery_client, dataset_id | |||
| 464 | 447 | assert tuple(table.schema) == table_schema | |
| 465 | 448 | assert table.num_rows == 3 | |
| 466 | 449 | ||
| 450 | + result = bigquery_client.list_rows(table).to_dataframe() | ||
| 451 | + result.sort_values("row_num", inplace=True) | ||
| 452 | + | ||
| 453 | + # Check that extreme DATE/DATETIME values are loaded correctly. | ||
| 454 | + # https://github.com/googleapis/python-bigquery/issues/1076 | ||
| 455 | + assert result["date_col"][0] == datetime.date(1, 1, 1) | ||
| 456 | + assert result["date_col"][2] == datetime.date(9999, 12, 31) | ||
| 457 | + assert result["dt_col"][0] == datetime.datetime(1, 1, 1, 0, 0, 0) | ||
| 458 | + assert result["dt_col"][2] == datetime.datetime(9999, 12, 31, 23, 59, 59, 999999) | ||
| 459 | + assert result["ts_col"][0] == datetime.datetime( | ||
| 460 | + 1, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc | ||
| 461 | + ) | ||
| 462 | + assert result["ts_col"][2] == datetime.datetime( | ||
| 463 | + 9999, 12, 31, 23, 59, 59, 999999, tzinfo=datetime.timezone.utc | ||
| 464 | + ) | ||
| 465 | + | ||
| 467 | 466 | ||
| 468 | 467 | def test_load_table_from_dataframe_w_struct_datatype(bigquery_client, dataset_id): | |
| 469 | 468 | """Test that a DataFrame with struct datatype can be uploaded if a | |
0 commit comments