From 0a844e08a1be5b91fa79c5dcc7203cf42c28bd84 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 28 Aug 2025 11:00:26 +0000 Subject: [PATCH] Workflow saved --- payment_metrics/main.py | 187 +++++++++++++++++++++++- payment_metrics/main.py.notebook | 239 +++++++++++++++++++++++++++++-- payment_metrics/main.workflow | 2 +- 3 files changed, 414 insertions(+), 14 deletions(-) diff --git a/payment_metrics/main.py b/payment_metrics/main.py index 509052b..8d41c7a 100644 --- a/payment_metrics/main.py +++ b/payment_metrics/main.py @@ -71,12 +71,12 @@ bootstrap_udfs(spark) -success_payments_df = spark.read.table('dremio.payments') -success_payments_df.createOrReplaceTempView('success_payments_df') +success_payments_reader_df = spark.read.table('dremio.payments') +success_payments_reader_df.createOrReplaceTempView('success_payments_reader_df') # %% -_success_payments_mapper_select_clause=success_payments_df.columns if False else [] +_success_payments_mapper_select_clause=success_payments_reader_df.columns if False else [] _success_payments_mapper_select_clause.append("DATE(payment_date) AS payment_date") @@ -87,7 +87,7 @@ _success_payments_mapper_select_clause.append("gateway AS gateway") _success_payments_mapper_select_clause.append("payment_method AS payment_method") -success_payments_mapper_df=spark.sql(("SELECT " + ', '.join(_success_payments_mapper_select_clause) + " FROM success_payments_df").replace("{job_id}",f"'{job_id}'")) +success_payments_mapper_df=spark.sql(("SELECT " + ', '.join(_success_payments_mapper_select_clause) + " FROM success_payments_reader_df").replace("{job_id}",f"'{job_id}'")) success_payments_mapper_df.createOrReplaceTempView("success_payments_mapper_df") # %% @@ -273,3 +273,182 @@ high_valued_payments_df.createOrReplaceTempView('high_valued_payments_df') + +# %% + + + +failed_payments_reader_df = spark.read.table('dremio.failedpayments') +failed_payments_reader_df.createOrReplaceTempView('failed_payments_reader_df') + +# %% + +_failed_payments_mapper_select_clause=failed_payments_reader_df.columns if False else [] + +_failed_payments_mapper_select_clause.append("DATE(payment_date) AS payment_date") + +_failed_payments_mapper_select_clause.append("payment_method AS payment_method") + +_failed_payments_mapper_select_clause.append("failure_reason AS failure_reason") + +_failed_payments_mapper_select_clause.append("gateway AS gateway") + + +failed_payments_mapper_df=spark.sql(("SELECT " + ', '.join(_failed_payments_mapper_select_clause) + " FROM failed_payments_reader_df").replace("{job_id}",f"'{job_id}'")) +failed_payments_mapper_df.createOrReplaceTempView("failed_payments_mapper_df") + +# %% + +print(failed_payments_mapper_df.columns) +final_failed_payments_df = spark.sql("select * from failed_payments_mapper_df where payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))") +final_failed_payments_df.createOrReplaceTempView('final_failed_payments_df') + +# %% + +print(final_failed_payments_df.columns) +filter__13_df = spark.sql("select * from final_failed_payments_df where gateway = \'CCS\'") +filter__13_df.createOrReplaceTempView('filter__13_df') + +# %% + + + + + + + + + +_params = { + "datasource": "filter__13", + "selectFunctions" : [{'fieldName': 'total_failed_payments', 'aggregationFunction': 'COUNT(*)'}] +} + +_df_flat, _grouping_specs, _rewritten_selects = preprocess_then_expand( filter__13_df, +group_expression="payment_date", +cube="", +rollup="", +grouping_set="", +select_functions=[{'fieldName': 'total_failed_payments', 'aggregationFunction': 'COUNT(*)'}] +) + +_agg_exprs = [expr(f["aggregationFunction"]).alias(f["fieldName"]) +for f in _rewritten_selects +] + +_all_group_cols = list({c for gs in _grouping_specs for c in gs}) + +_partials = [] +for _gs in _grouping_specs: + _gdf = _df_flat.groupBy(*_gs).agg(*_agg_exprs) + for _col in _all_group_cols: + if _col not in _gs: + _gdf = _gdf.withColumn(_col, lit(None)) + _partials.append(_gdf) + + +total_failed_payments___df = reduce(lambda a, b: a.unionByName(b), _partials) + +total_failed_payments___df.createOrReplaceTempView('total_failed_payments___df') + + + + + +# %% + + + + + + + + + +_params = { + "datasource": "final_failed_payments", + "selectFunctions" : [{'fieldName': 'failure_count', 'aggregationFunction': 'COUNT(*)'}] +} + +_df_flat, _grouping_specs, _rewritten_selects = preprocess_then_expand( final_failed_payments_df, +group_expression="payment_date, gateway, failure_reason", +cube="", +rollup="", +grouping_set="", +select_functions=[{'fieldName': 'failure_count', 'aggregationFunction': 'COUNT(*)'}] +) + +_agg_exprs = [expr(f["aggregationFunction"]).alias(f["fieldName"]) +for f in _rewritten_selects +] + +_all_group_cols = list({c for gs in _grouping_specs for c in gs}) + +_partials = [] +for _gs in _grouping_specs: + _gdf = _df_flat.groupBy(*_gs).agg(*_agg_exprs) + for _col in _all_group_cols: + if _col not in _gs: + _gdf = _gdf.withColumn(_col, lit(None)) + _partials.append(_gdf) + + +failed_payment_metrics_df = reduce(lambda a, b: a.unionByName(b), _partials) + +failed_payment_metrics_df.createOrReplaceTempView('failed_payment_metrics_df') + + + + + +# %% + + + + +_data_writer__15_fields_to_update = failed_payment_metrics_df.columns +_data_writer__15_set_clause=[] +_data_writer__15_unique_key_clause= [] + +for _key in ['payment_date', 'gateway', 'failure_reason']: + _data_writer__15_unique_key_clause.append(f't.{_key} = s.{_key}') + +for _field in _data_writer__15_fields_to_update: + if(_field not in _data_writer__15_unique_key_clause): + _data_writer__15_set_clause.append(f't.{_field} = s.{_field}') + +_merge_query = ''' + MERGE INTO dremio.failedpaymentmetrics t + USING failed_payment_metrics_df s + ON ''' + ' AND '.join(_data_writer__15_unique_key_clause) + ''' WHEN MATCHED THEN + UPDATE SET ''' + ', '.join(_data_writer__15_set_clause) + ' WHEN NOT MATCHED THEN INSERT *' + +spark.sql(_merge_query) + + + +# %% + +print(total_payments_and_total_value_processed_df.columns) +print(most_used_payment_method___df.columns) +print(high_valued_payments_df.columns) +print(total_failed_payments___df.columns) + +success_payment_metrics_df = spark.sql(""" +SELECT + COALESCE(a.payment_date, d.payment_date) AS payment_date, + a.total_payments, + a.total_value_processed, + b.most_used_payment_method, + c.high_valued_payments, + d.total_failed_payments +FROM total_failed_payments___df d +FULL OUTER JOIN total_payments_and_total_value_processed_df a + ON a.payment_date = d.payment_date +LEFT JOIN most_used_payment_method___df b + ON a.payment_date = b.payment_date +LEFT JOIN high_valued_payments_df c + ON a.payment_date = c.payment_date +""") + +success_payment_metrics_df.createOrReplaceTempView('success_payment_metrics_df') diff --git a/payment_metrics/main.py.notebook b/payment_metrics/main.py.notebook index bec9c5b..054f473 100644 --- a/payment_metrics/main.py.notebook +++ b/payment_metrics/main.py.notebook @@ -74,19 +74,19 @@ def init(): @app.cell -def success_payments(spark): +def success_payments_reader(spark): - success_payments_df = spark.read.table('dremio.payments') - success_payments_df.createOrReplaceTempView('success_payments_df') - return (success_payments_df,) + success_payments_reader_df = spark.read.table('dremio.payments') + success_payments_reader_df.createOrReplaceTempView('success_payments_reader_df') + return (success_payments_reader_df,) @app.cell -def success_payments_mapper(job_id, spark, success_payments_df): +def success_payments_mapper(job_id, spark, success_payments_reader_df): - _success_payments_mapper_select_clause=success_payments_df.columns if False else [] + _success_payments_mapper_select_clause=success_payments_reader_df.columns if False else [] _success_payments_mapper_select_clause.append("DATE(payment_date) AS payment_date") @@ -97,7 +97,7 @@ def success_payments_mapper(job_id, spark, success_payments_df): _success_payments_mapper_select_clause.append("payment_method AS payment_method") - success_payments_mapper_df=spark.sql(("SELECT " + ', '.join(_success_payments_mapper_select_clause) + " FROM success_payments_df").replace("{job_id}",f"'{job_id}'")) + success_payments_mapper_df=spark.sql(("SELECT " + ', '.join(_success_payments_mapper_select_clause) + " FROM success_payments_reader_df").replace("{job_id}",f"'{job_id}'")) success_payments_mapper_df.createOrReplaceTempView("success_payments_mapper_df") return (success_payments_mapper_df,) @@ -172,7 +172,7 @@ def total_payments_and_total_value_processed( - return + return (total_payments_and_total_value_processed_df,) @app.cell @@ -270,7 +270,7 @@ def most_used_payment_method__(filter__6_df, job_id, spark): most_used_payment_method___df=spark.sql(("SELECT " + ', '.join(_most_used_payment_method___select_clause) + " FROM filter__6_df").replace("{job_id}",f"'{job_id}'")) most_used_payment_method___df.createOrReplaceTempView("most_used_payment_method___df") - return + return (most_used_payment_method___df,) @app.cell @@ -325,6 +325,227 @@ def high_valued_payments( + return (high_valued_payments_df,) + + +@app.cell +def failed_payments_reader(spark): + + + + failed_payments_reader_df = spark.read.table('dremio.failedpayments') + failed_payments_reader_df.createOrReplaceTempView('failed_payments_reader_df') + return (failed_payments_reader_df,) + + +@app.cell +def failed_payments_mapper(failed_payments_reader_df, job_id, spark): + + _failed_payments_mapper_select_clause=failed_payments_reader_df.columns if False else [] + + _failed_payments_mapper_select_clause.append("DATE(payment_date) AS payment_date") + + _failed_payments_mapper_select_clause.append("payment_method AS payment_method") + + _failed_payments_mapper_select_clause.append("failure_reason AS failure_reason") + + _failed_payments_mapper_select_clause.append("gateway AS gateway") + + + failed_payments_mapper_df=spark.sql(("SELECT " + ', '.join(_failed_payments_mapper_select_clause) + " FROM failed_payments_reader_df").replace("{job_id}",f"'{job_id}'")) + failed_payments_mapper_df.createOrReplaceTempView("failed_payments_mapper_df") + return (failed_payments_mapper_df,) + + +@app.cell +def final_failed_payments(failed_payments_mapper_df, spark): + + print(failed_payments_mapper_df.columns) + final_failed_payments_df = spark.sql("select * from failed_payments_mapper_df where payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))") + final_failed_payments_df.createOrReplaceTempView('final_failed_payments_df') + return (final_failed_payments_df,) + + +@app.cell +def filter__13(final_failed_payments_df, spark): + + print(final_failed_payments_df.columns) + filter__13_df = spark.sql("select * from final_failed_payments_df where gateway = \'CCS\'") + filter__13_df.createOrReplaceTempView('filter__13_df') + return (filter__13_df,) + + +@app.cell +def total_failed_payments__( + expr, + filter__13_df, + lit, + preprocess_then_expand, + reduce, +): + + + + + + + + + + _params = { + "datasource": "filter__13", + "selectFunctions" : [{'fieldName': 'total_failed_payments', 'aggregationFunction': 'COUNT(*)'}] + } + + _df_flat, _grouping_specs, _rewritten_selects = preprocess_then_expand( filter__13_df, + group_expression="payment_date", + cube="", + rollup="", + grouping_set="", + select_functions=[{'fieldName': 'total_failed_payments', 'aggregationFunction': 'COUNT(*)'}] + ) + + _agg_exprs = [expr(f["aggregationFunction"]).alias(f["fieldName"]) + for f in _rewritten_selects + ] + + _all_group_cols = list({c for gs in _grouping_specs for c in gs}) + + _partials = [] + for _gs in _grouping_specs: + _gdf = _df_flat.groupBy(*_gs).agg(*_agg_exprs) + for _col in _all_group_cols: + if _col not in _gs: + _gdf = _gdf.withColumn(_col, lit(None)) + _partials.append(_gdf) + + + total_failed_payments___df = reduce(lambda a, b: a.unionByName(b), _partials) + + total_failed_payments___df.createOrReplaceTempView('total_failed_payments___df') + + + + + return (total_failed_payments___df,) + + +@app.cell +def failed_payment_metrics( + expr, + final_failed_payments_df, + lit, + preprocess_then_expand, + reduce, +): + + + + + + + + + + _params = { + "datasource": "final_failed_payments", + "selectFunctions" : [{'fieldName': 'failure_count', 'aggregationFunction': 'COUNT(*)'}] + } + + _df_flat, _grouping_specs, _rewritten_selects = preprocess_then_expand( final_failed_payments_df, + group_expression="payment_date, gateway, failure_reason", + cube="", + rollup="", + grouping_set="", + select_functions=[{'fieldName': 'failure_count', 'aggregationFunction': 'COUNT(*)'}] + ) + + _agg_exprs = [expr(f["aggregationFunction"]).alias(f["fieldName"]) + for f in _rewritten_selects + ] + + _all_group_cols = list({c for gs in _grouping_specs for c in gs}) + + _partials = [] + for _gs in _grouping_specs: + _gdf = _df_flat.groupBy(*_gs).agg(*_agg_exprs) + for _col in _all_group_cols: + if _col not in _gs: + _gdf = _gdf.withColumn(_col, lit(None)) + _partials.append(_gdf) + + + failed_payment_metrics_df = reduce(lambda a, b: a.unionByName(b), _partials) + + failed_payment_metrics_df.createOrReplaceTempView('failed_payment_metrics_df') + + + + + return (failed_payment_metrics_df,) + + +@app.cell +def data_writer__15(failed_payment_metrics_df, spark): + + + + + _data_writer__15_fields_to_update = failed_payment_metrics_df.columns + _data_writer__15_set_clause=[] + _data_writer__15_unique_key_clause= [] + + for _key in ['payment_date', 'gateway', 'failure_reason']: + _data_writer__15_unique_key_clause.append(f't.{_key} = s.{_key}') + + for _field in _data_writer__15_fields_to_update: + if(_field not in _data_writer__15_unique_key_clause): + _data_writer__15_set_clause.append(f't.{_field} = s.{_field}') + + _merge_query = ''' + MERGE INTO dremio.failedpaymentmetrics t + USING failed_payment_metrics_df s + ON ''' + ' AND '.join(_data_writer__15_unique_key_clause) + ''' WHEN MATCHED THEN + UPDATE SET ''' + ', '.join(_data_writer__15_set_clause) + ' WHEN NOT MATCHED THEN INSERT *' + + spark.sql(_merge_query) + + + return + + +@app.cell +def success_payment_metrics( + high_valued_payments_df, + most_used_payment_method___df, + spark, + total_failed_payments___df, + total_payments_and_total_value_processed_df, +): + + print(total_payments_and_total_value_processed_df.columns) + print(most_used_payment_method___df.columns) + print(high_valued_payments_df.columns) + print(total_failed_payments___df.columns) + + success_payment_metrics_df = spark.sql(""" + SELECT + COALESCE(a.payment_date, d.payment_date) AS payment_date, + a.total_payments, + a.total_value_processed, + b.most_used_payment_method, + c.high_valued_payments, + d.total_failed_payments + FROM total_failed_payments___df d + FULL OUTER JOIN total_payments_and_total_value_processed_df a + ON a.payment_date = d.payment_date + LEFT JOIN most_used_payment_method___df b + ON a.payment_date = b.payment_date + LEFT JOIN high_valued_payments_df c + ON a.payment_date = c.payment_date + """) + + success_payment_metrics_df.createOrReplaceTempView('success_payment_metrics_df') return diff --git a/payment_metrics/main.workflow b/payment_metrics/main.workflow index 12b98d9..242147b 100644 --- a/payment_metrics/main.workflow +++ b/payment_metrics/main.workflow @@ -1 +1 @@ -{"version":"v1alpha","kind":"VisualBuilder","metadata":{"name":"payment_metrics","description":"Workflow to generate payment metrics.","runtime":"spark"},"spec":{"ui":{"edges":[{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__1","target":"data-mapper__0","id":"xy-edge__data-reader__1-data-mapper__0"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__0","target":"filter__1","id":"xy-edge__data-mapper__0-filter__1"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"filter__3","id":"xy-edge__filter__1-filter__3"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__4","id":"xy-edge__filter__1-aggregate__4"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__7","id":"xy-edge__filter__1-aggregate__7"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__4","target":"data-mapper__5","id":"xy-edge__aggregate__4-data-mapper__5"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__5","target":"filter__6","id":"xy-edge__data-mapper__5-filter__6"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__6","target":"data-mapper__8","id":"xy-edge__filter__6-data-mapper__8"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__3","target":"aggregate__9","id":"xy-edge__filter__3-aggregate__9"}],"nodes":[{"id":"data-reader__1","type":"workflowNode","position":{"x":-1768.5340546476618,"y":-162.4753522546897},"data":{"nodeType":"data-reader","id":"data-reader__1"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__0","type":"workflowNode","position":{"x":-1353.8681752505147,"y":-169.04851494308113},"data":{"nodeType":"data-mapper","id":"data-mapper__0"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__1","type":"workflowNode","position":{"x":-921.2468878928117,"y":-170.40923466009258},"data":{"nodeType":"filter","id":"filter__1"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__3","type":"workflowNode","position":{"x":-429.06026348805125,"y":120.4585301507131},"data":{"nodeType":"filter","id":"filter__3"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__4","type":"workflowNode","position":{"x":-408.1850633096815,"y":-543.9835681697932},"data":{"nodeType":"aggregate","id":"aggregate__4"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__5","type":"workflowNode","position":{"x":175.81493669031852,"y":-549.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__5"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__6","type":"workflowNode","position":{"x":767.8149366903185,"y":-547.9835681697932},"data":{"nodeType":"filter","id":"filter__6"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__7","type":"workflowNode","position":{"x":1393.8149366903185,"y":-165.98356816979322},"data":{"nodeType":"aggregate","id":"aggregate__7"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__8","type":"workflowNode","position":{"x":1281.8149366903185,"y":-555.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__8"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__9","type":"workflowNode","position":{"x":1413.8149366903185,"y":100.01643183020678},"data":{"nodeType":"aggregate","id":"aggregate__9"},"measured":{"width":320,"height":110},"selected":true,"dragging":false},{"id":"data-reader__10","type":"workflowNode","position":{"x":-1798.1850633096815,"y":568.0164318302068},"data":{"nodeType":"data-reader","id":"data-reader__10"},"measured":{"width":320,"height":142},"selected":false,"dragging":false},{"id":"data-mapper__11","type":"workflowNode","position":{"x":-1374.1850633096815,"y":566.0164318302068},"data":{"nodeType":"data-mapper","id":"data-mapper__11"},"measured":{"width":320,"height":142},"selected":false,"dragging":false},{"id":"filter__12","type":"workflowNode","position":{"x":-930.1850633096815,"y":584.0164318302068},"data":{"nodeType":"filter","id":"filter__12"},"measured":{"width":320,"height":97},"selected":false,"dragging":false},{"id":"filter__13","type":"workflowNode","position":{"x":-414.1850633096815,"y":390.0164318302068},"data":{"nodeType":"filter","id":"filter__13"},"measured":{"width":320,"height":97},"selected":false,"dragging":false},{"id":"aggregate__14","type":"workflowNode","position":{"x":-422.1850633096815,"y":718.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__14"},"measured":{"width":320,"height":120},"selected":false,"dragging":false},{"id":"data-writer__15","type":"workflowNode","position":{"x":183.81493669031852,"y":716.0164318302068},"data":{"nodeType":"data-writer","id":"data-writer__15"},"measured":{"width":320,"height":142},"selected":false,"dragging":false},{"id":"aggregate__16","type":"workflowNode","position":{"x":1415.8149366903185,"y":360.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__16"},"measured":{"width":320,"height":120},"selected":false,"dragging":false},{"id":"code-transform__17","type":"workflowNode","position":{"x":2037.8149366903185,"y":-109.98356816979322},"data":{"nodeType":"code-transform","id":"code-transform__17"},"measured":{"width":320,"height":142},"selected":false,"dragging":false},{"id":"data-writer__18","type":"workflowNode","position":{"x":2617.8149366903185,"y":-109.98356816979322},"data":{"nodeType":"data-writer","id":"data-writer__18"},"measured":{"width":320,"height":142},"selected":false,"dragging":false}],"nodesData":{"data-reader__1":{"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","name":"success_payments","type":"SparkIcebergReader","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__0":{"name":"success_payments_mapper","type":"DataMapping","fromDataReader":"success_payments","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__1":{"name":"final_success_payments","type":"Filter","datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},"filter__3":{"name":"high_valued_payments_filter","type":"Filter","datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},"aggregate__7":{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},"aggregate__4":{"name":"aggregate__4","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-mapper__5":{"name":"data_mapper__5","type":"DataMapping","fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__6":{"name":"filter__6","type":"Filter","datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},"data-mapper__8":{"name":"most_used_payment_method__","type":"DataMapping","fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"aggregate__9":{"name":"high_valued_payments","type":"SQLAggregation","datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false}}},"blocks":[{"name":"success_payments","type":"SparkIcebergReader","options":{},"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"success_payments_mapper","type":"DataMapping","options":{},"fromDataReader":"success_payments","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"final_success_payments","type":"Filter","options":{},"datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},{"name":"high_valued_payments_filter","type":"Filter","options":{},"datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},{"name":"aggregate__4","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"data_mapper__5","type":"DataMapping","options":{},"fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"filter__6","type":"Filter","options":{},"datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},{"name":"most_used_payment_method__","type":"DataMapping","options":{},"fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"high_valued_payments","type":"SQLAggregation","options":{},"datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false}]}} \ No newline at end of file +{"version":"v1alpha","kind":"VisualBuilder","metadata":{"name":"payment_metrics","description":"Workflow to generate payment metrics.","runtime":"spark"},"spec":{"ui":{"edges":[{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__1","target":"data-mapper__0","id":"xy-edge__data-reader__1-data-mapper__0"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__0","target":"filter__1","id":"xy-edge__data-mapper__0-filter__1"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"filter__3","id":"xy-edge__filter__1-filter__3"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__4","id":"xy-edge__filter__1-aggregate__4"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__1","target":"aggregate__7","id":"xy-edge__filter__1-aggregate__7"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__4","target":"data-mapper__5","id":"xy-edge__aggregate__4-data-mapper__5"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__5","target":"filter__6","id":"xy-edge__data-mapper__5-filter__6"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__6","target":"data-mapper__8","id":"xy-edge__filter__6-data-mapper__8"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__3","target":"aggregate__9","id":"xy-edge__filter__3-aggregate__9"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__10","target":"data-mapper__11","id":"xy-edge__data-reader__10-data-mapper__11"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__11","target":"filter__12","id":"xy-edge__data-mapper__11-filter__12"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__12","target":"filter__13","id":"xy-edge__filter__12-filter__13"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__12","target":"aggregate__14","id":"xy-edge__filter__12-aggregate__14"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__13","target":"aggregate__16","id":"xy-edge__filter__13-aggregate__16"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__14","target":"data-writer__15","id":"xy-edge__aggregate__14-data-writer__15"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__8","target":"code-transform__17","id":"xy-edge__data-mapper__8-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__7","target":"code-transform__17","id":"xy-edge__aggregate__7-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__9","target":"code-transform__17","id":"xy-edge__aggregate__9-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__16","target":"code-transform__17","id":"xy-edge__aggregate__16-code-transform__17"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"code-transform__17","target":"data-writer__18","id":"xy-edge__code-transform__17-data-writer__18"}],"nodes":[{"id":"data-reader__1","type":"workflowNode","position":{"x":-1768.5340546476618,"y":-162.4753522546897},"data":{"nodeType":"data-reader","id":"data-reader__1"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__0","type":"workflowNode","position":{"x":-1353.8681752505147,"y":-169.04851494308113},"data":{"nodeType":"data-mapper","id":"data-mapper__0"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__1","type":"workflowNode","position":{"x":-921.2468878928117,"y":-170.40923466009258},"data":{"nodeType":"filter","id":"filter__1"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__3","type":"workflowNode","position":{"x":-429.06026348805125,"y":120.4585301507131},"data":{"nodeType":"filter","id":"filter__3"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__4","type":"workflowNode","position":{"x":-408.1850633096815,"y":-543.9835681697932},"data":{"nodeType":"aggregate","id":"aggregate__4"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__5","type":"workflowNode","position":{"x":175.81493669031852,"y":-549.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__5"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__6","type":"workflowNode","position":{"x":767.8149366903185,"y":-547.9835681697932},"data":{"nodeType":"filter","id":"filter__6"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__7","type":"workflowNode","position":{"x":1393.8149366903185,"y":-177.98356816979322},"data":{"nodeType":"aggregate","id":"aggregate__7"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__8","type":"workflowNode","position":{"x":1383.8149366903185,"y":-549.9835681697932},"data":{"nodeType":"data-mapper","id":"data-mapper__8"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__9","type":"workflowNode","position":{"x":1393.8149366903185,"y":120.01643183020678},"data":{"nodeType":"aggregate","id":"aggregate__9"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-reader__10","type":"workflowNode","position":{"x":-1798.1850633096815,"y":580.0164318302068},"data":{"nodeType":"data-reader","id":"data-reader__10"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__11","type":"workflowNode","position":{"x":-1366.1850633096815,"y":576.0164318302068},"data":{"nodeType":"data-mapper","id":"data-mapper__11"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__12","type":"workflowNode","position":{"x":-930.1850633096815,"y":584.0164318302068},"data":{"nodeType":"filter","id":"filter__12"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__13","type":"workflowNode","position":{"x":-416.1850633096815,"y":390.0164318302068},"data":{"nodeType":"filter","id":"filter__13"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__14","type":"workflowNode","position":{"x":-410.1850633096815,"y":756.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__14"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-writer__15","type":"workflowNode","position":{"x":181.81493669031852,"y":736.0164318302068},"data":{"nodeType":"data-writer","id":"data-writer__15"},"measured":{"width":320,"height":138},"selected":false,"dragging":false},{"id":"aggregate__16","type":"workflowNode","position":{"x":1381.8149366903185,"y":378.0164318302068},"data":{"nodeType":"aggregate","id":"aggregate__16"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"code-transform__17","type":"workflowNode","position":{"x":2037.8149366903185,"y":-109.98356816979322},"data":{"nodeType":"code-transform","id":"code-transform__17"},"measured":{"width":320,"height":110},"selected":true,"dragging":false},{"id":"data-writer__18","type":"workflowNode","position":{"x":2609.8149366903185,"y":-129.98356816979322},"data":{"nodeType":"data-writer","id":"data-writer__18"},"measured":{"width":320,"height":142},"selected":false,"dragging":false}],"nodesData":{"data-reader__1":{"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","name":"success_payments_reader","type":"SparkIcebergReader","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__0":{"name":"success_payments_mapper","type":"DataMapping","fromDataReader":"success_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__1":{"name":"final_success_payments","type":"Filter","datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},"filter__3":{"name":"high_valued_payments_filter","type":"Filter","datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},"aggregate__7":{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},"aggregate__4":{"name":"aggregate__4","type":"SQLAggregation","datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-mapper__5":{"name":"data_mapper__5","type":"DataMapping","fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__6":{"name":"filter__6","type":"Filter","datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},"data-mapper__8":{"name":"most_used_payment_method__","type":"DataMapping","fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"aggregate__9":{"name":"high_valued_payments","type":"SQLAggregation","datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-reader__10":{"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"name":"failed_payments_reader","type":"SparkIcebergReader","table_name":"failedpayments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__11":{"name":"failed_payments_mapper","type":"DataMapping","fromDataReader":"failed_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"failure_reason","valueExpression":"failure_reason"},{"fieldName":"gateway","valueExpression":"gateway"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756377601359","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756377617744","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756377625433","newFieldName":"failure_reason","mappingType":"sourceColumn","value":"failure_reason"},{"id":"mapping-1756377634290","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__12":{"name":"final_failed_payments","type":"Filter","datasource":"failed_payments_mapper","condition":"payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))","isDefault":false},"filter__13":{"name":"filter__13","type":"Filter","datasource":"final_failed_payments","condition":"gateway = \\'CCS\\'","isDefault":false},"aggregate__16":{"name":"total_failed_payments__","type":"SQLAggregation","datasource":"filter__13","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_failed_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},"aggregate__14":{"name":"failed_payment_metrics","type":"SQLAggregation","datasource":"final_failed_payments","groupByParams":{"group_expression":"payment_date, gateway, failure_reason"},"selectFunctions":[{"fieldName":"failure_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},"data-writer__15":{"name":"data_writer__15","type":"IcebergWriter","iceberg_catalog":"dremio","warehouse_directory":"failedpaymentmetrics","datasource":"failed_payment_metrics","mode":"merge","typeLabel":"Iceberg","unique_id":["payment_date","gateway","failure_reason"],"isDefault":false},"code-transform__17":{"name":"success_payment_metrics","type":"CodeTransform","code":"print(total_payments_and_total_value_processed_df.columns)\nprint(most_used_payment_method___df.columns)\nprint(high_valued_payments_df.columns)\nprint(total_failed_payments___df.columns)\n\n{{name}}_df = spark.sql(\"\"\"\nSELECT \n COALESCE(a.payment_date, d.payment_date) AS payment_date,\n a.total_payments,\n a.total_value_processed,\n b.most_used_payment_method,\n c.high_valued_payments,\n d.total_failed_payments\nFROM total_failed_payments___df d\nFULL OUTER JOIN total_payments_and_total_value_processed_df a \n ON a.payment_date = d.payment_date\nLEFT JOIN most_used_payment_method___df b \n ON a.payment_date = b.payment_date\nLEFT JOIN high_valued_payments_df c \n ON a.payment_date = c.payment_date\n\"\"\")\n\n{{name}}_df.createOrReplaceTempView('{{name}}_df')","isDefault":false}}},"blocks":[{"name":"success_payments_reader","type":"SparkIcebergReader","options":{},"isDefault":false,"iceberg_catalog":"dremio","typeLabel":"Spark","table_name":"payments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"success_payments_mapper","type":"DataMapping","options":{},"fromDataReader":"success_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"amount","valueExpression":"amount"},{"fieldName":"gateway","valueExpression":"gateway"},{"fieldName":"payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756374444833","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756374453565","newFieldName":"amount","mappingType":"sourceColumn","value":"amount"},{"id":"mapping-1756374468084","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"},{"id":"mapping-1756374477932","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"final_success_payments","type":"Filter","options":{},"datasource":"success_payments_mapper","condition":" payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.successpaymentmetrics), (SELECT MIN(payment_date) FROM success_payments_mapper_df)) AND gateway = \\'CCS\\'","isDefault":false},{"name":"high_valued_payments_filter","type":"Filter","options":{},"datasource":"final_success_payments","condition":"amount >= 500","isDefault":false},{"name":"total_payments_and_total_value_processed","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_payments","aggregationFunction":"COUNT(*)"},{"fieldName":"total_value_processed","aggregationFunction":"SUM(amount)"}],"isDefault":false},{"name":"aggregate__4","type":"SQLAggregation","options":{},"datasource":"final_success_payments","groupByParams":{"group_expression":"payment_date, payment_method"},"selectFunctions":[{"fieldName":"method_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"data_mapper__5","type":"DataMapping","options":{},"fromDataReader":"aggregate__4","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"method_count","valueExpression":"method_count"},{"fieldName":"rank_method","valueExpression":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375393396","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375408077","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756375415696","newFieldName":"method_count","mappingType":"sourceColumn","value":"method_count"},{"id":"mapping-1756375432743","newFieldName":"rank_method","mappingType":"expression","value":"RANK() OVER (PARTITION BY payment_date ORDER BY method_count)"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"filter__6","type":"Filter","options":{},"datasource":"data_mapper__5","condition":"rank_method = 1","isDefault":false},{"name":"most_used_payment_method__","type":"DataMapping","options":{},"fromDataReader":"filter__6","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"payment_date"},{"fieldName":"most_used_payment_method","valueExpression":"payment_method"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756375574506","newFieldName":"payment_date","mappingType":"sourceColumn","value":"payment_date"},{"id":"mapping-1756375594556","newFieldName":"most_used_payment_method","mappingType":"sourceColumn","value":"payment_method"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"high_valued_payments","type":"SQLAggregation","options":{},"datasource":"high_valued_payments_filter","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"high_valued_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"failed_payments_reader","type":"SparkIcebergReader","options":{},"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"table_name":"failedpayments","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"failed_payments_mapper","type":"DataMapping","options":{},"fromDataReader":"failed_payments_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"payment_date","valueExpression":"DATE(payment_date)"},{"fieldName":"payment_method","valueExpression":"payment_method"},{"fieldName":"failure_reason","valueExpression":"failure_reason"},{"fieldName":"gateway","valueExpression":"gateway"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756377601359","newFieldName":"payment_date","mappingType":"expression","value":"DATE(payment_date)"},{"id":"mapping-1756377617744","newFieldName":"payment_method","mappingType":"sourceColumn","value":"payment_method"},{"id":"mapping-1756377625433","newFieldName":"failure_reason","mappingType":"sourceColumn","value":"failure_reason"},{"id":"mapping-1756377634290","newFieldName":"gateway","mappingType":"sourceColumn","value":"gateway"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"final_failed_payments","type":"Filter","options":{},"datasource":"failed_payments_mapper","condition":"payment_date >= COALESCE((SELECT MAX(DATE(payment_date)) FROM dremio.failedpaymentmetrics), (SELECT MIN(payment_date) FROM failed_payments_mapper_df))","isDefault":false},{"name":"filter__13","type":"Filter","options":{},"datasource":"final_failed_payments","condition":"gateway = \\'CCS\\'","isDefault":false},{"name":"total_failed_payments__","type":"SQLAggregation","options":{},"datasource":"filter__13","groupByParams":{"group_expression":"payment_date"},"selectFunctions":[{"fieldName":"total_failed_payments","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"failed_payment_metrics","type":"SQLAggregation","options":{},"datasource":"final_failed_payments","groupByParams":{"group_expression":"payment_date, gateway, failure_reason"},"selectFunctions":[{"fieldName":"failure_count","aggregationFunction":"COUNT(*)"}],"isDefault":false},{"name":"data_writer__15","type":"IcebergWriter","options":{},"iceberg_catalog":"dremio","warehouse_directory":"failedpaymentmetrics","datasource":"failed_payment_metrics","mode":"merge","typeLabel":"Iceberg","unique_id":["payment_date","gateway","failure_reason"],"isDefault":false},{"name":"success_payment_metrics","type":"CodeTransform","options":{},"code":"print(total_payments_and_total_value_processed_df.columns)\nprint(most_used_payment_method___df.columns)\nprint(high_valued_payments_df.columns)\nprint(total_failed_payments___df.columns)\n\n{{name}}_df = spark.sql(\"\"\"\nSELECT \n COALESCE(a.payment_date, d.payment_date) AS payment_date,\n a.total_payments,\n a.total_value_processed,\n b.most_used_payment_method,\n c.high_valued_payments,\n d.total_failed_payments\nFROM total_failed_payments___df d\nFULL OUTER JOIN total_payments_and_total_value_processed_df a \n ON a.payment_date = d.payment_date\nLEFT JOIN most_used_payment_method___df b \n ON a.payment_date = b.payment_date\nLEFT JOIN high_valued_payments_df c \n ON a.payment_date = c.payment_date\n\"\"\")\n\n{{name}}_df.createOrReplaceTempView('{{name}}_df')","isDefault":false}]}} \ No newline at end of file