1 line
24 KiB
XML
1 line
24 KiB
XML
{"version":"v1alpha","kind":"VisualBuilder","metadata":{"name":"payment_metrics","description":"Workflow to generate payment metrics.","runtime":"spark"},"spec":{"ui":{"edges":[{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__1","target":"data-mapper__0","id":"xy-edge__data-reader__1-data-mapper__0"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__0","target":"filter__1","id":"xy-edge__data-mapper__0-filter__1"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"filter__3","id":"xy-edge__filter__1-filter__3"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__4","id":"xy-edge__filter__1-aggregate__4"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__7","id":"xy-edge__filter__1-aggregate__7"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__4","target":"data-mapper__5","id":"xy-edge__aggregate__4-data-mapper__5"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__5","target":"filter__6","id":"xy-edge__data-mapper__5-filter__6"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__6","target":"data-mapper__8","id":"xy-edge__filter__6-data-mapper__8"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__3","target":"aggregate__9","id":"xy-edge__filter__3-aggregate__9"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__10","target":"data-mapper__11","id":"xy-edge__data-reader__10-data-mapper__11"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__11","target":"filter__12","id":"xy-edge__data-mapper__11-filter__12"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__12","target":"filter__13","id":"xy-edge__filter__12-filter__13"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__12","target":"aggregate__14","id":"xy-edge__filter__12-aggregate__14"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__13","target":"aggregate__16","id":"xy-edge__filter__13-aggregate__16"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__14","target":"data-writer__15","id":"xy-edge__aggregate__14-data-writer__15"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__8","target":"code-transform__17","id":"xy-edge__data-mapper__8-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__7","target":"code-transform__17","id":"xy-edge__aggregate__7-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__9","target":"code-transform__17","id":"xy-edge__aggregate__9-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__16","target":"code-transform__17","id":"xy-edge__aggregate__16-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"code-transform__17","target":"data-writer__18","id":"xy-edge__code-transform__17-data-writer__18"}],"nodes":[{"id":"data-reader__1","type":"workflowNode","position":{"x":-1768.5340546476618,"y":-162.4753522546897},"data":{"nodeType":"data-reader","id":"data-reader__1"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__0","type":"workflowNode","position":{"x":-1353.8681752505147,"y":-169.04851494308113},"data":{"nodeType":"data-mapper","id":"data-mapper__0"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__1","type":"workflowNode","position":{"x":-921.2468878928117,"y":-170.40923466009258},"data":{"nodeType":"filter","id":"filter__1"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__3","type":"workflowNode","position":{"x":-429.06026348805125,"y":120.4585301507131},"data":{"nodeType":"filter","id":"filter__3"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__4","type":"workflowNode","position":{"x":-408.1850633096815,"y":-543.9835681697932},"data":{"nodeType":"aggregate","id":"aggregate__4"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__5","type":"workflowNode","position":{"x":175.81493669031852,"y":-549.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__5"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__6","type":"workflowNode","position":{"x":767.8149366903185,"y":-547.9835681697932},"data":{"nodeType":"filter","id":"filter__6"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__7","type":"workflowNode","position":{"x":1393.8149366903185,"y":-177.98356816979322},"data":{"nodeType":"aggregate","id":"aggregate__7"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__8","type":"workflowNode","position":{"x":1383.8149366903185,"y":-549.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__8"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__9","type":"workflowNode","position":{"x":1393.8149366903185,"y":120.01643183020678},"data":{"nodeType":"aggregate","id":"aggregate__9"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-reader__10","type":"workflowNode","position":{"x":-1798.1850633096815,"y":580.0164318302068},"data":{"nodeType":"data-reader","id":"data-reader__10"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__11","type":"workflowNode","position":{"x":-1366.1850633096815,"y":576.0164318302068},"data":{"nodeType":"data-mapper","id":"data-mapper__11"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__12","type":"workflowNode","position":{"x":-930.1850633096815,"y":584.0164318302068},"data":{"nodeType":"filter","id":"filter__12"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__13","type":"workflowNode","position":{"x":-416.1850633096815,"y":390.0164318302068},"data":{"nodeType":"filter","id":"filter__13"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__14","type":"workflowNode","position":{"x":-410.1850633096815,"y":756.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__14"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-writer__15","type":"workflowNode","position":{"x":181.81493669031852,"y":736.0164318302068},"data":{"nodeType":"data-writer","id":"data-writer__15"},"measured":{"width":320,"height":138},"selected":false,"dragging":false},{"id":"aggregate__16","type":"workflowNode","position":{"x":1381.8149366903185,"y":378.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__16"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"code-transform__17","type":"workflowNode","position":{"x":2045.8149366903185,"y":-57.983568169793216},"data":{"nodeType":"code-transform","id":"code-transform__17"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-writer__18","type":"workflowNode","position":{"x":2611.8149366903185,"y":-81.98356816979322},"data":{"nodeType":"data-writer","id":"data-writer__18"},"measured":{"width":320,"height":142},"selected":true,"dragging":false}],"nodesData":{"data-reader__1":{"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","name":"success_payments_reader","type":"SparkIcebergReader","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__0":{"name":"success_payments_mapper","type":"DataMapping","fromDataReader":"success_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__1":{"name":"final_success_payments","type":"Filter","datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},"filter__3":{"name":"high_valued_payments_filter","type":"Filter","datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},"aggregate__7":{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},"aggregate__4":{"name":"aggregate__4","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-mapper__5":{"name":"data_mapper__5","type":"DataMapping","fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__6":{"name":"filter__6","type":"Filter","datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},"data-mapper__8":{"name":"most_used_payment_method__","type":"DataMapping","fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"aggregate__9":{"name":"high_valued_payments__","type":"SQLAggregation","datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-reader__10":{"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"name":"failed_payments_reader","type":"SparkIcebergReader","table_name":"failedpayments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__11":{"name":"failed_payments_mapper","type":"DataMapping","fromDataReader":"failed_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"failure_reason","valueExpression":"failure_reason"},{"fieldName":"gateway","valueExpression":"gateway"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756377601359","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756377617744","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756377625433","newFieldName":"failure_reason","mappingType":"sourceColumn","value":"failure_reason"},{"id":"mapping-1756377634290","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__12":{"name":"final_failed_payments","type":"Filter","datasource":"failed_payments_mapper","condition":"payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))","isDefault":false},"filter__13":{"name":"filter__13","type":"Filter","datasource":"final_failed_payments","condition":"gateway = \\'CCS\\'","isDefault":false},"aggregate__16":{"name":"total_failed_payments__","type":"SQLAggregation","datasource":"filter__13","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_failed_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},"aggregate__14":{"name":"failed_payment_metrics","type":"SQLAggregation","datasource":"final_failed_payments","groupByParams":{"group_expression":"payment_date, gateway, failure_reason"},"selectFunctions":[{"fieldName":"failure_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-writer__15":{"name":"data_writer__15","type":"IcebergWriter","iceberg_catalog":"dremio","warehouse_directory":"failedpaymentmetrics","datasource":"failed_payment_metrics","mode":"merge","typeLabel":"Iceberg","unique_id":["payment_date","gateway","failure_reason"],"isDefault":false},"code-transform__17":{"name":"success_payment_metrics","type":"CodeTransform","code":"print(total_payments_and_total_value_processed_df.columns)\nprint(most_used_payment_method___df.columns)\nprint(high_valued_payments___df.columns)\nprint(total_failed_payments___df.columns)\n\n{{name}}_df = spark.sql(\"\"\"\nSELECT \n COALESCE(a.payment_date, d.payment_date) AS payment_date,\n a.total_payments,\n a.total_value_processed,\n b.most_used_payment_method,\n c.high_valued_payments,\n d.total_failed_payments\nFROM total_failed_payments___df d\nFULL OUTER JOIN total_payments_and_total_value_processed_df a \n ON a.payment_date = d.payment_date\nLEFT JOIN most_used_payment_method___df b \n ON a.payment_date = b.payment_date\nLEFT JOIN high_valued_payments___df c \n ON a.payment_date = c.payment_date\n\"\"\")\n\n{{name}}_df.createOrReplaceTempView('{{name}}_df')","isDefault":false}}},"blocks":[{"name":"success_payments_reader","type":"SparkIcebergReader","options":{},"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"success_payments_mapper","type":"DataMapping","options":{},"fromDataReader":"success_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"final_success_payments","type":"Filter","options":{},"datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},{"name":"high_valued_payments_filter","type":"Filter","options":{},"datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},{"name":"aggregate__4","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"data_mapper__5","type":"DataMapping","options":{},"fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"filter__6","type":"Filter","options":{},"datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},{"name":"most_used_payment_method__","type":"DataMapping","options":{},"fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"high_valued_payments__","type":"SQLAggregation","options":{},"datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"failed_payments_reader","type":"SparkIcebergReader","options":{},"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"table_name":"failedpayments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"failed_payments_mapper","type":"DataMapping","options":{},"fromDataReader":"failed_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"failure_reason","valueExpression":"failure_reason"},{"fieldName":"gateway","valueExpression":"gateway"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756377601359","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756377617744","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756377625433","newFieldName":"failure_reason","mappingType":"sourceColumn","value":"failure_reason"},{"id":"mapping-1756377634290","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"final_failed_payments","type":"Filter","options":{},"datasource":"failed_payments_mapper","condition":"payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))","isDefault":false},{"name":"filter__13","type":"Filter","options":{},"datasource":"final_failed_payments","condition":"gateway = \\'CCS\\'","isDefault":false},{"name":"total_failed_payments__","type":"SQLAggregation","options":{},"datasource":"filter__13","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_failed_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"failed_payment_metrics","type":"SQLAggregation","options":{},"datasource":"final_failed_payments","groupByParams":{"group_expression":"payment_date, gateway, failure_reason"},"selectFunctions":[{"fieldName":"failure_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"data_writer__15","type":"IcebergWriter","options":{},"iceberg_catalog":"dremio","warehouse_directory":"failedpaymentmetrics","datasource":"failed_payment_metrics","mode":"merge","typeLabel":"Iceberg","unique_id":["payment_date","gateway","failure_reason"],"isDefault":false},{"name":"success_payment_metrics","type":"CodeTransform","options":{},"code":"print(total_payments_and_total_value_processed_df.columns)\nprint(most_used_payment_method___df.columns)\nprint(high_valued_payments___df.columns)\nprint(total_failed_payments___df.columns)\n\n{{name}}_df = spark.sql(\"\"\"\nSELECT \n COALESCE(a.payment_date, d.payment_date) AS payment_date,\n a.total_payments,\n a.total_value_processed,\n b.most_used_payment_method,\n c.high_valued_payments,\n d.total_failed_payments\nFROM total_failed_payments___df d\nFULL OUTER JOIN total_payments_and_total_value_processed_df a \n ON a.payment_date = d.payment_date\nLEFT JOIN most_used_payment_method___df b \n ON a.payment_date = b.payment_date\nLEFT JOIN high_valued_payments___df c \n ON a.payment_date = c.payment_date\n\"\"\")\n\n{{name}}_df.createOrReplaceTempView('{{name}}_df')","isDefault":false}]}} |