From ec5e61ce9e81e9106f62cf09116c2597a93939a8 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 19 Sep 2024 18:24:58 -0700 Subject: [PATCH 1/3] Rewrite tests. --- lib/parquetutil/parse_values_test.go | 137 ++++++++++++--------------- 1 file changed, 62 insertions(+), 75 deletions(-) diff --git a/lib/parquetutil/parse_values_test.go b/lib/parquetutil/parse_values_test.go index 1cc5ab3c..abf83449 100644 --- a/lib/parquetutil/parse_values_test.go +++ b/lib/parquetutil/parse_values_test.go @@ -13,84 +13,71 @@ import ( ) func TestParseValue(t *testing.T) { - eDecimal := typing.NewDecimalDetailsFromTemplate(typing.EDecimal, decimal.NewDetails(30, 5)) - - eTime := typing.ETime - eTime.ExtendedTimeDetails = &ext.Time - - eDate := typing.ETime - eDate.ExtendedTimeDetails = &ext.Date - - eDateTime := typing.ETime - eDateTime.ExtendedTimeDetails = &ext.TimestampTz - - testCases := []struct { - name string - colVal any - colKind columns.Column - - expectedValue any - }{ - { - name: "nil value", - colVal: nil, - expectedValue: nil, - }, - { - name: "string value", - colVal: "test", - colKind: columns.NewColumn("", typing.String), - expectedValue: "test", - }, - { - name: "struct value", - colVal: map[string]any{ - "foo": "bar", - }, - colKind: columns.NewColumn("", typing.Struct), - expectedValue: `{"foo":"bar"}`, - }, - { - name: "array (numbers - converted to string)", - colVal: []any{123, 456}, - colKind: columns.NewColumn("", typing.Array), - expectedValue: []string{"123", "456"}, - }, - { - name: "array (boolean - converted to string)", - colVal: []any{true, false, true}, - colKind: columns.NewColumn("", typing.Array), - expectedValue: []string{"true", "false", "true"}, - }, - { - name: "decimal", - colVal: decimal.NewDecimalWithPrecision(numbers.MustParseDecimal("5000.22320"), 30), - colKind: columns.NewColumn("", eDecimal), - expectedValue: "5000.22320", - }, - { - name: "time", - colVal: "03:15:00", - colKind: columns.NewColumn("", eTime), - expectedValue: "03:15:00+00", - }, + { + // Nil + value, err := ParseValue(nil, columns.Column{}) + assert.NoError(t, err) + assert.Nil(t, value) + } + { + // String + value, err := ParseValue("test", columns.NewColumn("", typing.String)) + assert.NoError(t, err) + assert.Equal(t, "test", value) + } + { + // Struct + value, err := ParseValue(map[string]any{"foo": "bar"}, columns.NewColumn("", typing.Struct)) + assert.NoError(t, err) + assert.Equal(t, `{"foo":"bar"}`, value) + } + { + // Arrays { - name: "date", - colVal: "2022-12-25", - colKind: columns.NewColumn("", eDate), - expectedValue: "2022-12-25", - }, + // Arrays (numbers - converted to string) + value, err := ParseValue([]any{123, 456}, columns.NewColumn("", typing.Array)) + assert.NoError(t, err) + assert.Equal(t, []string{"123", "456"}, value) + } { - name: "datetime", - colVal: "2023-04-24T17:29:05.69944Z", - colKind: columns.NewColumn("", eDateTime), - expectedValue: int64(1682357345699), - }, + // Arrays (booleans - converted to string) + value, err := ParseValue([]any{false, true, false}, columns.NewColumn("", typing.Array)) + assert.NoError(t, err) + assert.Equal(t, []string{"false", "true", "false"}, value) + } } + { + // Decimal + value, err := ParseValue(decimal.NewDecimalWithPrecision( + numbers.MustParseDecimal("5000.22320"), 30), + columns.NewColumn("", typing.NewDecimalDetailsFromTemplate(typing.EDecimal, decimal.NewDetails(30, 5))), + ) - for _, tc := range testCases { - actualValue, actualErr := ParseValue(tc.colVal, tc.colKind) - assert.NoError(t, actualErr, tc.name) - assert.Equal(t, tc.expectedValue, actualValue, tc.name) + assert.NoError(t, err) + assert.Equal(t, "5000.22320", value) + } + { + // Time + eTime := typing.ETime + eTime.ExtendedTimeDetails = &ext.Time + value, err := ParseValue("03:15:00", columns.NewColumn("", eTime)) + assert.NoError(t, err) + assert.Equal(t, "03:15:00+00", value) + } + { + // Date + eDate := typing.ETime + eDate.ExtendedTimeDetails = &ext.Date + value, err := ParseValue("2022-12-25", columns.NewColumn("", eDate)) + assert.NoError(t, err) + assert.Equal(t, "2022-12-25", value) + } + { + // Timestamp TZ + eDateTime := typing.ETime + eDateTime.ExtendedTimeDetails = &ext.TimestampTz + value, err := ParseValue("2023-04-24T17:29:05.69944Z", columns.NewColumn("", eDateTime)) + assert.NoError(t, err) + assert.Equal(t, int64(1682357345699), value) } } From e193d62dc41b3bb1907ece9c8c3f70bc2fa19800 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 19 Sep 2024 18:25:21 -0700 Subject: [PATCH 2/3] Imports. --- lib/parquetutil/parse_values_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/parquetutil/parse_values_test.go b/lib/parquetutil/parse_values_test.go index abf83449..3c1be9eb 100644 --- a/lib/parquetutil/parse_values_test.go +++ b/lib/parquetutil/parse_values_test.go @@ -4,11 +4,10 @@ import ( "testing" "github.com/artie-labs/transfer/lib/numbers" - "github.com/artie-labs/transfer/lib/typing/ext" - "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/columns" "github.com/artie-labs/transfer/lib/typing/decimal" + "github.com/artie-labs/transfer/lib/typing/ext" "github.com/stretchr/testify/assert" ) From 5bc2cc2c5a1d7b71377ba3b9d1206a9a8881f038 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 19 Sep 2024 18:27:17 -0700 Subject: [PATCH 3/3] Another. --- lib/kafkalib/partition/settings_test.go | 96 +++++++++---------------- 1 file changed, 33 insertions(+), 63 deletions(-) diff --git a/lib/kafkalib/partition/settings_test.go b/lib/kafkalib/partition/settings_test.go index 75ea7f03..209af8eb 100644 --- a/lib/kafkalib/partition/settings_test.go +++ b/lib/kafkalib/partition/settings_test.go @@ -7,69 +7,39 @@ import ( ) func TestBigQuerySettings_Valid(t *testing.T) { - testCases := []struct { - name string - bigQuerySettings *BigQuerySettings - expectedErr string - }{ - { - name: "nil", - expectedErr: "bigQuerySettings is nil", - }, - { - name: "empty partitionType", - bigQuerySettings: &BigQuerySettings{}, - expectedErr: "partitionTypes cannot be empty", - }, - { - name: "empty partitionField", - bigQuerySettings: &BigQuerySettings{ - PartitionType: "time", - }, - expectedErr: "partitionField cannot be empty", - }, - { - name: "empty partitionBy", - bigQuerySettings: &BigQuerySettings{ - PartitionType: "time", - PartitionField: "created_at", - }, - expectedErr: "partitionBy cannot be empty", - }, - { - name: "invalid partitionType", - bigQuerySettings: &BigQuerySettings{ - PartitionType: "invalid", - PartitionField: "created_at", - PartitionBy: "daily", - }, - expectedErr: "partitionType must be one of:", - }, - { - name: "invalid partitionBy", - bigQuerySettings: &BigQuerySettings{ - PartitionType: "time", - PartitionField: "created_at", - PartitionBy: "invalid", - }, - expectedErr: "partitionBy must be one of:", - }, - { - name: "valid", - bigQuerySettings: &BigQuerySettings{ - PartitionType: "time", - PartitionField: "created_at", - PartitionBy: "daily", - }, - }, + { + // Nil + var settings *BigQuerySettings + assert.ErrorContains(t, settings.Valid(), "bigQuerySettings is nil") } - - for _, testCase := range testCases { - actualErr := testCase.bigQuerySettings.Valid() - if testCase.expectedErr != "" { - assert.ErrorContains(t, actualErr, testCase.expectedErr, testCase.name) - } else { - assert.NoError(t, actualErr, testCase.name) - } + { + // Empty partition type + settings := &BigQuerySettings{} + assert.ErrorContains(t, settings.Valid(), "partitionTypes cannot be empty") + } + { + // Empty partition field + settings := &BigQuerySettings{PartitionType: "time"} + assert.ErrorContains(t, settings.Valid(), "partitionField cannot be empty") + } + { + // Empty partition by + settings := &BigQuerySettings{PartitionType: "time", PartitionField: "created_at"} + assert.ErrorContains(t, settings.Valid(), "partitionBy cannot be empty") + } + { + // Invalid partition type + settings := &BigQuerySettings{PartitionType: "invalid", PartitionField: "created_at", PartitionBy: "daily"} + assert.ErrorContains(t, settings.Valid(), "partitionType must be one of:") + } + { + // Invalid partition by + settings := &BigQuerySettings{PartitionType: "time", PartitionField: "created_at", PartitionBy: "invalid"} + assert.ErrorContains(t, settings.Valid(), "partitionBy must be one of:") + } + { + // Valid + settings := &BigQuerySettings{PartitionType: "time", PartitionField: "created_at", PartitionBy: "daily"} + assert.NoError(t, settings.Valid()) } }