diff --git a/payment_metrics/main.py b/payment_metrics/main.py index 7358b28..ce34f7a 100644 --- a/payment_metrics/main.py +++ b/payment_metrics/main.py @@ -299,12 +299,6 @@ failed_payments_mapper_df.createOrReplaceTempView("failed_payments_mapper_df") # %% -print(failed_payments_mapper_df.columns) -final_failed_payments_df = spark.sql("select * from failed_payments_mapper_df where payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))") -final_failed_payments_df.createOrReplaceTempView('final_failed_payments_df') - -# %% - print(final_failed_payments_df.columns) filter__13_df = spark.sql("select * from final_failed_payments_df where gateway = \'CCS\'") filter__13_df.createOrReplaceTempView('filter__13_df') @@ -478,3 +472,10 @@ _merge_query = ''' spark.sql(_merge_query) + +# %% + +print(FailedPaymentsData_df.columns) +LatestFailedPayments_df = spark.sql("select * from FailedPaymentsData_df where payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))") +failed_payments_df.createOrReplaceTempView('failed_payments_df') +failed_payments_df.persist() diff --git a/payment_metrics/main.py.notebook b/payment_metrics/main.py.notebook index ae9169b..b6d4293 100644 --- a/payment_metrics/main.py.notebook +++ b/payment_metrics/main.py.notebook @@ -354,16 +354,7 @@ def failed_payments_mapper(failed_payments_reader_df, job_id, spark): failed_payments_mapper_df=spark.sql(("SELECT " + ', '.join(_failed_payments_mapper_select_clause) + " FROM failed_payments_reader_df").replace("{job_id}",f"'{job_id}'")) failed_payments_mapper_df.createOrReplaceTempView("failed_payments_mapper_df") - return (failed_payments_mapper_df,) - - -@app.cell -def final_failed_payments(failed_payments_mapper_df, spark): - - print(failed_payments_mapper_df.columns) - final_failed_payments_df = spark.sql("select * from failed_payments_mapper_df where payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))") - final_failed_payments_df.createOrReplaceTempView('final_failed_payments_df') - return (final_failed_payments_df,) + return @app.cell @@ -578,5 +569,15 @@ def success_payment_metrics_writer(spark, success_payment_metrics_df): return +@app.cell +def failed_payments(FailedPaymentsData_df, failed_payments_df, spark): + + print(FailedPaymentsData_df.columns) + LatestFailedPayments_df = spark.sql("select * from FailedPaymentsData_df where payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))") + failed_payments_df.createOrReplaceTempView('failed_payments_df') + failed_payments_df.persist() + return + + if __name__ == "__main__": app.run() diff --git a/payment_metrics/main.workflow b/payment_metrics/main.workflow index bd3f397..8f1e882 100644 --- a/payment_metrics/main.workflow +++ b/payment_metrics/main.workflow @@ -1 +1 @@ -{"version":"v1alpha","kind":"VisualBuilder","metadata":{"name":"payment_metrics","description":"Workflow to generate payment metrics.","runtime":"spark"},"spec":{"ui":{"edges":[{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__1","target":"data-mapper__0","id":"xy-edge__data-reader__1-data-mapper__0"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__0","target":"filter__1","id":"xy-edge__data-mapper__0-filter__1"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"filter__3","id":"xy-edge__filter__1-filter__3"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__4","id":"xy-edge__filter__1-aggregate__4"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__7","id":"xy-edge__filter__1-aggregate__7"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__4","target":"data-mapper__5","id":"xy-edge__aggregate__4-data-mapper__5"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__5","target":"filter__6","id":"xy-edge__data-mapper__5-filter__6"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__6","target":"data-mapper__8","id":"xy-edge__filter__6-data-mapper__8"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__3","target":"aggregate__9","id":"xy-edge__filter__3-aggregate__9"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__10","target":"data-mapper__11","id":"xy-edge__data-reader__10-data-mapper__11"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__11","target":"filter__12","id":"xy-edge__data-mapper__11-filter__12"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__12","target":"filter__13","id":"xy-edge__filter__12-filter__13"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__12","target":"aggregate__14","id":"xy-edge__filter__12-aggregate__14"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__13","target":"aggregate__16","id":"xy-edge__filter__13-aggregate__16"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__14","target":"data-writer__15","id":"xy-edge__aggregate__14-data-writer__15"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__8","target":"code-transform__17","id":"xy-edge__data-mapper__8-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__7","target":"code-transform__17","id":"xy-edge__aggregate__7-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__9","target":"code-transform__17","id":"xy-edge__aggregate__9-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__16","target":"code-transform__17","id":"xy-edge__aggregate__16-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"code-transform__17","target":"data-writer__18","id":"xy-edge__code-transform__17-data-writer__18"}],"nodes":[{"id":"data-reader__1","type":"workflowNode","position":{"x":-1768.5340546476618,"y":-162.4753522546897},"data":{"nodeType":"data-reader","id":"data-reader__1"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__0","type":"workflowNode","position":{"x":-1353.8681752505147,"y":-169.04851494308113},"data":{"nodeType":"data-mapper","id":"data-mapper__0"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__1","type":"workflowNode","position":{"x":-921.2468878928117,"y":-170.40923466009258},"data":{"nodeType":"filter","id":"filter__1"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__3","type":"workflowNode","position":{"x":-429.06026348805125,"y":120.4585301507131},"data":{"nodeType":"filter","id":"filter__3"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__4","type":"workflowNode","position":{"x":-408.1850633096815,"y":-543.9835681697932},"data":{"nodeType":"aggregate","id":"aggregate__4"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__5","type":"workflowNode","position":{"x":175.81493669031852,"y":-549.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__5"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__6","type":"workflowNode","position":{"x":767.8149366903185,"y":-547.9835681697932},"data":{"nodeType":"filter","id":"filter__6"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__7","type":"workflowNode","position":{"x":1393.8149366903185,"y":-177.98356816979322},"data":{"nodeType":"aggregate","id":"aggregate__7"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__8","type":"workflowNode","position":{"x":1383.8149366903185,"y":-549.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__8"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__9","type":"workflowNode","position":{"x":1393.8149366903185,"y":120.01643183020678},"data":{"nodeType":"aggregate","id":"aggregate__9"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-reader__10","type":"workflowNode","position":{"x":-1798.1850633096815,"y":580.0164318302068},"data":{"nodeType":"data-reader","id":"data-reader__10"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__11","type":"workflowNode","position":{"x":-1366.1850633096815,"y":576.0164318302068},"data":{"nodeType":"data-mapper","id":"data-mapper__11"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__12","type":"workflowNode","position":{"x":-930.1850633096815,"y":584.0164318302068},"data":{"nodeType":"filter","id":"filter__12"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__13","type":"workflowNode","position":{"x":-416.1850633096815,"y":390.0164318302068},"data":{"nodeType":"filter","id":"filter__13"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__14","type":"workflowNode","position":{"x":-410.1850633096815,"y":756.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__14"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-writer__15","type":"workflowNode","position":{"x":181.81493669031852,"y":736.0164318302068},"data":{"nodeType":"data-writer","id":"data-writer__15"},"measured":{"width":320,"height":138},"selected":false,"dragging":false},{"id":"aggregate__16","type":"workflowNode","position":{"x":1381.8149366903185,"y":378.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__16"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"code-transform__17","type":"workflowNode","position":{"x":2045.8149366903185,"y":-57.983568169793216},"data":{"nodeType":"code-transform","id":"code-transform__17"},"measured":{"width":320,"height":110},"selected":true,"dragging":false},{"id":"data-writer__18","type":"workflowNode","position":{"x":2611.8149366903185,"y":-81.98356816979322},"data":{"nodeType":"data-writer","id":"data-writer__18"},"measured":{"width":320,"height":138},"selected":false,"dragging":false}],"nodesData":{"data-reader__1":{"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","name":"success_payments_reader","type":"SparkIcebergReader","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__0":{"name":"success_payments_mapper","type":"DataMapping","fromDataReader":"success_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__1":{"name":"final_success_payments","type":"Filter","datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},"filter__3":{"name":"high_valued_payments_filter","type":"Filter","datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},"aggregate__7":{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},"aggregate__4":{"name":"aggregate__4","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-mapper__5":{"name":"data_mapper__5","type":"DataMapping","fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__6":{"name":"filter__6","type":"Filter","datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},"data-mapper__8":{"name":"most_used_payment_method__","type":"DataMapping","fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"aggregate__9":{"name":"high_valued_payments__","type":"SQLAggregation","datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-reader__10":{"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"name":"failed_payments_reader","type":"SparkIcebergReader","table_name":"failedpayments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__11":{"name":"failed_payments_mapper","type":"DataMapping","fromDataReader":"failed_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"failure_reason","valueExpression":"failure_reason"},{"fieldName":"gateway","valueExpression":"gateway"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756377601359","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756377617744","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756377625433","newFieldName":"failure_reason","mappingType":"sourceColumn","value":"failure_reason"},{"id":"mapping-1756377634290","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__12":{"name":"final_failed_payments","type":"Filter","datasource":"failed_payments_mapper","condition":"payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))","isDefault":false},"filter__13":{"name":"filter__13","type":"Filter","datasource":"final_failed_payments","condition":"gateway = \\'CCS\\'","isDefault":false},"aggregate__16":{"name":"total_failed_payments__","type":"SQLAggregation","datasource":"filter__13","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_failed_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},"aggregate__14":{"name":"failed_payment_metrics","type":"SQLAggregation","datasource":"final_failed_payments","groupByParams":{"group_expression":"payment_date, gateway, failure_reason"},"selectFunctions":[{"fieldName":"failure_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-writer__15":{"name":"data_writer__15","type":"IcebergWriter","iceberg_catalog":"dremio","warehouse_directory":"failedpaymentmetrics","datasource":"failed_payment_metrics","mode":"merge","typeLabel":"Iceberg","unique_id":["payment_date","gateway","failure_reason"],"isDefault":false},"code-transform__17":{"name":"success_payment_metrics","type":"CodeTransform","code":"print(total_payments_and_total_value_processed_df.columns)\nprint(most_used_payment_method___df.columns)\nprint(high_valued_payments___df.columns)\nprint(total_failed_payments___df.columns)\n\n{{name}}_df = spark.sql(\"\"\"\nSELECT \n COALESCE(a.payment_date, d.payment_date) AS payment_date,\n a.total_payments,\n a.total_value_processed,\n b.most_used_payment_method,\n c.high_valued_payments,\n d.total_failed_payments\nFROM total_failed_payments___df d\nFULL OUTER JOIN total_payments_and_total_value_processed_df a \n ON a.payment_date = d.payment_date\nLEFT JOIN most_used_payment_method___df b \n ON a.payment_date = b.payment_date\nLEFT JOIN high_valued_payments___df c \n ON a.payment_date = c.payment_date\n\"\"\")\n\n{{name}}_df.createOrReplaceTempView('{{name}}_df')","isDefault":false},"data-writer__18":{"name":"success_payment_metrics_writer","type":"IcebergWriter","iceberg_catalog":"dremio","warehouse_directory":"successpaymentmetrics","datasource":"success_payment_metrics","mode":"merge","typeLabel":"Iceberg (Legacy)","unique_id":["payment_date"],"isDefault":false}}},"blocks":[{"name":"success_payments_reader","type":"SparkIcebergReader","options":{},"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"success_payments_mapper","type":"DataMapping","options":{},"fromDataReader":"success_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"final_success_payments","type":"Filter","options":{},"datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},{"name":"high_valued_payments_filter","type":"Filter","options":{},"datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},{"name":"aggregate__4","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"data_mapper__5","type":"DataMapping","options":{},"fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"filter__6","type":"Filter","options":{},"datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},{"name":"most_used_payment_method__","type":"DataMapping","options":{},"fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"high_valued_payments__","type":"SQLAggregation","options":{},"datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"failed_payments_reader","type":"SparkIcebergReader","options":{},"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"table_name":"failedpayments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"failed_payments_mapper","type":"DataMapping","options":{},"fromDataReader":"failed_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"failure_reason","valueExpression":"failure_reason"},{"fieldName":"gateway","valueExpression":"gateway"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756377601359","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756377617744","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756377625433","newFieldName":"failure_reason","mappingType":"sourceColumn","value":"failure_reason"},{"id":"mapping-1756377634290","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"final_failed_payments","type":"Filter","options":{},"datasource":"failed_payments_mapper","condition":"payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))","isDefault":false},{"name":"filter__13","type":"Filter","options":{},"datasource":"final_failed_payments","condition":"gateway = \\'CCS\\'","isDefault":false},{"name":"total_failed_payments__","type":"SQLAggregation","options":{},"datasource":"filter__13","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_failed_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"failed_payment_metrics","type":"SQLAggregation","options":{},"datasource":"final_failed_payments","groupByParams":{"group_expression":"payment_date, gateway, failure_reason"},"selectFunctions":[{"fieldName":"failure_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"data_writer__15","type":"IcebergWriter","options":{},"iceberg_catalog":"dremio","warehouse_directory":"failedpaymentmetrics","datasource":"failed_payment_metrics","mode":"merge","typeLabel":"Iceberg","unique_id":["payment_date","gateway","failure_reason"],"isDefault":false},{"name":"success_payment_metrics","type":"CodeTransform","options":{},"code":"print(total_payments_and_total_value_processed_df.columns)\nprint(most_used_payment_method___df.columns)\nprint(high_valued_payments___df.columns)\nprint(total_failed_payments___df.columns)\n\n{{name}}_df = spark.sql(\"\"\"\nSELECT \n COALESCE(a.payment_date, d.payment_date) AS payment_date,\n a.total_payments,\n a.total_value_processed,\n b.most_used_payment_method,\n c.high_valued_payments,\n d.total_failed_payments\nFROM total_failed_payments___df d\nFULL OUTER JOIN total_payments_and_total_value_processed_df a \n ON a.payment_date = d.payment_date\nLEFT JOIN most_used_payment_method___df b \n ON a.payment_date = b.payment_date\nLEFT JOIN high_valued_payments___df c \n ON a.payment_date = c.payment_date\n\"\"\")\n\n{{name}}_df.createOrReplaceTempView('{{name}}_df')","isDefault":false},{"name":"success_payment_metrics_writer","type":"IcebergWriter","options":{},"iceberg_catalog":"dremio","warehouse_directory":"successpaymentmetrics","datasource":"success_payment_metrics","mode":"merge","typeLabel":"Iceberg (Legacy)","unique_id":["payment_date"],"isDefault":false}]}} \ No newline at end of file +{"version":"v1alpha","kind":"VisualBuilder","metadata":{"name":"payment_metrics","description":"Workflow to generate payment metrics.","runtime":"spark"},"spec":{"ui":{"edges":[{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__1","target":"data-mapper__0","id":"xy-edge__data-reader__1-data-mapper__0"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__0","target":"filter__1","id":"xy-edge__data-mapper__0-filter__1"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"filter__3","id":"xy-edge__filter__1-filter__3"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__4","id":"xy-edge__filter__1-aggregate__4"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__7","id":"xy-edge__filter__1-aggregate__7"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__4","target":"data-mapper__5","id":"xy-edge__aggregate__4-data-mapper__5"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__5","target":"filter__6","id":"xy-edge__data-mapper__5-filter__6"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__6","target":"data-mapper__8","id":"xy-edge__filter__6-data-mapper__8"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__3","target":"aggregate__9","id":"xy-edge__filter__3-aggregate__9"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__10","target":"data-mapper__11","id":"xy-edge__data-reader__10-data-mapper__11"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__13","target":"aggregate__16","id":"xy-edge__filter__13-aggregate__16"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__14","target":"data-writer__15","id":"xy-edge__aggregate__14-data-writer__15"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__8","target":"code-transform__17","id":"xy-edge__data-mapper__8-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__7","target":"code-transform__17","id":"xy-edge__aggregate__7-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__9","target":"code-transform__17","id":"xy-edge__aggregate__9-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__16","target":"code-transform__17","id":"xy-edge__aggregate__16-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"code-transform__17","target":"data-writer__18","id":"xy-edge__code-transform__17-data-writer__18"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__11","target":"code-transform__0","id":"xy-edge__data-mapper__11-code-transform__0"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"code-transform__0","target":"filter__13","id":"xy-edge__code-transform__0-filter__13"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"code-transform__0","target":"aggregate__14","id":"xy-edge__code-transform__0-aggregate__14"}],"nodes":[{"id":"data-reader__1","type":"workflowNode","position":{"x":-1762.5340546476618,"y":-168.4753522546897},"data":{"nodeType":"data-reader","id":"data-reader__1"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"data-mapper__0","type":"workflowNode","position":{"x":-1353.8681752505147,"y":-169.04851494308113},"data":{"nodeType":"data-mapper","id":"data-mapper__0"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"filter__1","type":"workflowNode","position":{"x":-921.2468878928117,"y":-170.40923466009258},"data":{"nodeType":"filter","id":"filter__1"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"filter__3","type":"workflowNode","position":{"x":-429.06026348805125,"y":120.4585301507131},"data":{"nodeType":"filter","id":"filter__3"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"aggregate__4","type":"workflowNode","position":{"x":-408.1850633096815,"y":-543.9835681697932},"data":{"nodeType":"aggregate","id":"aggregate__4"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"data-mapper__5","type":"workflowNode","position":{"x":175.81493669031852,"y":-549.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__5"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"filter__6","type":"workflowNode","position":{"x":767.8149366903185,"y":-547.9835681697932},"data":{"nodeType":"filter","id":"filter__6"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"aggregate__7","type":"workflowNode","position":{"x":1393.8149366903185,"y":-177.98356816979322},"data":{"nodeType":"aggregate","id":"aggregate__7"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"data-mapper__8","type":"workflowNode","position":{"x":1383.8149366903185,"y":-549.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__8"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"aggregate__9","type":"workflowNode","position":{"x":1393.8149366903185,"y":120.01643183020678},"data":{"nodeType":"aggregate","id":"aggregate__9"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"data-reader__10","type":"workflowNode","position":{"x":-1798.1850633096815,"y":580.0164318302068},"data":{"nodeType":"data-reader","id":"data-reader__10"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"data-mapper__11","type":"workflowNode","position":{"x":-1362.1850633096815,"y":580.0164318302068},"data":{"nodeType":"data-mapper","id":"data-mapper__11"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"filter__13","type":"workflowNode","position":{"x":-416.1850633096815,"y":390.0164318302068},"data":{"nodeType":"filter","id":"filter__13"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"aggregate__14","type":"workflowNode","position":{"x":-410.1850633096815,"y":756.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__14"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"data-writer__15","type":"workflowNode","position":{"x":181.81493669031852,"y":736.0164318302068},"data":{"nodeType":"data-writer","id":"data-writer__15"},"measured":{"width":320,"height":139},"selected":false,"dragging":false},{"id":"aggregate__16","type":"workflowNode","position":{"x":1381.8149366903185,"y":378.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__16"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"code-transform__17","type":"workflowNode","position":{"x":2045.8149366903185,"y":-57.983568169793216},"data":{"nodeType":"code-transform","id":"code-transform__17"},"measured":{"width":320,"height":111},"selected":false,"dragging":false},{"id":"data-writer__18","type":"workflowNode","position":{"x":2611.8149366903185,"y":-81.98356816979322},"data":{"nodeType":"data-writer","id":"data-writer__18"},"measured":{"width":320,"height":139},"selected":false,"dragging":false},{"id":"code-transform__0","type":"workflowNode","position":{"x":-928.2552865705557,"y":564.8811573007712},"data":{"nodeType":"code-transform","id":"code-transform__0"},"measured":{"width":320,"height":111},"selected":true,"dragging":false}],"nodesData":{"data-reader__1":{"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","name":"success_payments_reader","type":"SparkIcebergReader","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__0":{"name":"success_payments_mapper","type":"DataMapping","fromDataReader":"success_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__1":{"name":"final_success_payments","type":"Filter","datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},"filter__3":{"name":"high_valued_payments_filter","type":"Filter","datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},"aggregate__7":{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},"aggregate__4":{"name":"aggregate__4","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-mapper__5":{"name":"data_mapper__5","type":"DataMapping","fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__6":{"name":"filter__6","type":"Filter","datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},"data-mapper__8":{"name":"most_used_payment_method__","type":"DataMapping","fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"aggregate__9":{"name":"high_valued_payments__","type":"SQLAggregation","datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-reader__10":{"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"name":"failed_payments_reader","type":"SparkIcebergReader","table_name":"failedpayments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__11":{"name":"failed_payments_mapper","type":"DataMapping","fromDataReader":"failed_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"failure_reason","valueExpression":"failure_reason"},{"fieldName":"gateway","valueExpression":"gateway"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756377601359","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756377617744","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756377625433","newFieldName":"failure_reason","mappingType":"sourceColumn","value":"failure_reason"},{"id":"mapping-1756377634290","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__13":{"name":"filter__13","type":"Filter","datasource":"final_failed_payments","condition":"gateway = \\'CCS\\'","isDefault":false},"aggregate__16":{"name":"total_failed_payments__","type":"SQLAggregation","datasource":"filter__13","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_failed_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},"aggregate__14":{"name":"failed_payment_metrics","type":"SQLAggregation","datasource":"final_failed_payments","groupByParams":{"group_expression":"payment_date, gateway, failure_reason"},"selectFunctions":[{"fieldName":"failure_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-writer__15":{"name":"data_writer__15","type":"IcebergWriter","iceberg_catalog":"dremio","warehouse_directory":"failedpaymentmetrics","datasource":"failed_payment_metrics","mode":"merge","typeLabel":"Iceberg","unique_id":["payment_date","gateway","failure_reason"],"isDefault":false},"code-transform__17":{"name":"success_payment_metrics","type":"CodeTransform","code":"print(total_payments_and_total_value_processed_df.columns)\nprint(most_used_payment_method___df.columns)\nprint(high_valued_payments___df.columns)\nprint(total_failed_payments___df.columns)\n\n{{name}}_df = spark.sql(\"\"\"\nSELECT \n COALESCE(a.payment_date, d.payment_date) AS payment_date,\n a.total_payments,\n a.total_value_processed,\n b.most_used_payment_method,\n c.high_valued_payments,\n d.total_failed_payments\nFROM total_failed_payments___df d\nFULL OUTER JOIN total_payments_and_total_value_processed_df a \n ON a.payment_date = d.payment_date\nLEFT JOIN most_used_payment_method___df b \n ON a.payment_date = b.payment_date\nLEFT JOIN high_valued_payments___df c \n ON a.payment_date = c.payment_date\n\"\"\")\n\n{{name}}_df.createOrReplaceTempView('{{name}}_df')","isDefault":false},"data-writer__18":{"name":"success_payment_metrics_writer","type":"IcebergWriter","iceberg_catalog":"dremio","warehouse_directory":"successpaymentmetrics","datasource":"success_payment_metrics","mode":"merge","typeLabel":"Iceberg (Legacy)","unique_id":["payment_date"],"isDefault":false},"code-transform__0":{"name":"failed_payments","type":"CodeTransform","language":"python","datasource":"","code":"print(FailedPaymentsData_df.columns)\nLatestFailedPayments_df = spark.sql(\"select * from FailedPaymentsData_df where payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))\")\n{{name}}_df.createOrReplaceTempView('failed_payments_df')\n{{name}}_df.persist()","isDefault":false,"connectedComponents":["failed_payments_mapper"]}}},"blocks":[{"name":"success_payments_reader","type":"SparkIcebergReader","options":{},"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"success_payments_mapper","type":"DataMapping","options":{},"fromDataReader":"success_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"final_success_payments","type":"Filter","options":{},"datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},{"name":"high_valued_payments_filter","type":"Filter","options":{},"datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},{"name":"aggregate__4","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"data_mapper__5","type":"DataMapping","options":{},"fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"filter__6","type":"Filter","options":{},"datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},{"name":"most_used_payment_method__","type":"DataMapping","options":{},"fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"high_valued_payments__","type":"SQLAggregation","options":{},"datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"failed_payments_reader","type":"SparkIcebergReader","options":{},"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"table_name":"failedpayments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"failed_payments_mapper","type":"DataMapping","options":{},"fromDataReader":"failed_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"failure_reason","valueExpression":"failure_reason"},{"fieldName":"gateway","valueExpression":"gateway"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756377601359","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756377617744","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756377625433","newFieldName":"failure_reason","mappingType":"sourceColumn","value":"failure_reason"},{"id":"mapping-1756377634290","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"filter__13","type":"Filter","options":{},"datasource":"final_failed_payments","condition":"gateway = \\'CCS\\'","isDefault":false},{"name":"total_failed_payments__","type":"SQLAggregation","options":{},"datasource":"filter__13","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_failed_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"failed_payment_metrics","type":"SQLAggregation","options":{},"datasource":"final_failed_payments","groupByParams":{"group_expression":"payment_date, gateway, failure_reason"},"selectFunctions":[{"fieldName":"failure_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"data_writer__15","type":"IcebergWriter","options":{},"iceberg_catalog":"dremio","warehouse_directory":"failedpaymentmetrics","datasource":"failed_payment_metrics","mode":"merge","typeLabel":"Iceberg","unique_id":["payment_date","gateway","failure_reason"],"isDefault":false},{"name":"success_payment_metrics","type":"CodeTransform","options":{},"code":"print(total_payments_and_total_value_processed_df.columns)\nprint(most_used_payment_method___df.columns)\nprint(high_valued_payments___df.columns)\nprint(total_failed_payments___df.columns)\n\n{{name}}_df = spark.sql(\"\"\"\nSELECT \n COALESCE(a.payment_date, d.payment_date) AS payment_date,\n a.total_payments,\n a.total_value_processed,\n b.most_used_payment_method,\n c.high_valued_payments,\n d.total_failed_payments\nFROM total_failed_payments___df d\nFULL OUTER JOIN total_payments_and_total_value_processed_df a \n ON a.payment_date = d.payment_date\nLEFT JOIN most_used_payment_method___df b \n ON a.payment_date = b.payment_date\nLEFT JOIN high_valued_payments___df c \n ON a.payment_date = c.payment_date\n\"\"\")\n\n{{name}}_df.createOrReplaceTempView('{{name}}_df')","isDefault":false},{"name":"success_payment_metrics_writer","type":"IcebergWriter","options":{},"iceberg_catalog":"dremio","warehouse_directory":"successpaymentmetrics","datasource":"success_payment_metrics","mode":"merge","typeLabel":"Iceberg (Legacy)","unique_id":["payment_date"],"isDefault":false},{"name":"failed_payments","type":"CodeTransform","options":{},"language":"python","datasource":"","code":"print(FailedPaymentsData_df.columns)\nLatestFailedPayments_df = spark.sql(\"select * from FailedPaymentsData_df where payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))\")\n{{name}}_df.createOrReplaceTempView('failed_payments_df')\n{{name}}_df.persist()","isDefault":false,"connectedComponents":["failed_payments_mapper"]}]}} \ No newline at end of file