tenant1/service_request_metrics/main.workflow

{"version":"v1alpha","kind":"VisualBuilder","metadata":{"name":"service_request_metrics","description":"Service request metrics workflow","runtime":"spark"},"spec":{"ui":{"edges":[{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__0","target":"data-mapper__1","id":"xy-edge__data-reader__0-data-mapper__1"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__1","target":"filter__2","id":"xy-edge__data-mapper__1-filter__2"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__2","target":"aggregate__3","id":"xy-edge__filter__2-aggregate__3"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__3","target":"data-writer__5","id":"xy-edge__aggregate__3-data-writer__5"}],"nodes":[{"id":"data-reader__0","type":"workflowNode","position":{"x":-1794,"y":-34},"data":{"nodeType":"data-reader","id":"data-reader__0"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-mapper__1","type":"workflowNode","position":{"x":-1347.5,"y":-36.499999999999886},"data":{"nodeType":"data-mapper","id":"data-mapper__1"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"filter__2","type":"workflowNode","position":{"x":-899.5,"y":-34.499999999999886},"data":{"nodeType":"filter","id":"filter__2"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"aggregate__3","type":"workflowNode","position":{"x":-473.5,"y":-32.499999999999886},"data":{"nodeType":"aggregate","id":"aggregate__3"},"measured":{"width":320,"height":110},"selected":false,"dragging":false},{"id":"data-writer__5","type":"workflowNode","position":{"x":-7.5,"y":-52.499999999999886},"data":{"nodeType":"data-writer","id":"data-writer__5"},"measured":{"width":320,"height":138},"selected":true,"dragging":false}],"nodesData":{"data-reader__0":{"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"name":"actions_audit_reader","type":"SparkIcebergReader","table_name":"actionsaudit","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"options":{}},"data-mapper__1":{"name":"actions_audit_mapper","type":"DataMapping","fromDataReader":"actions_audit_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"action_date","valueExpression":"DATE(action_date)"},{"fieldName":"service_type","valueExpression":"sub_category"},{"fieldName":"action_count","valueExpression":"action_count"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756445643483","newFieldName":"action_date","mappingType":"expression","value":"DATE(action_date)"},{"id":"mapping-1756445668299","newFieldName":"service_type","mappingType":"sourceColumn","value":"sub_category"},{"id":"mapping-1756445681282","newFieldName":"action_count","mappingType":"sourceColumn","value":"action_count"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},"filter__2":{"name":"actions_audit_filter","type":"Filter","datasource":"actions_audit_mapper","condition":"action_date >=  COALESCE((SELECT MAX(DATE(action_date)) FROM dremio.servicemetrics), (SELECT MIN(action_date) FROM actions_audit_mapper_df))","isDefault":false},"aggregate__3":{"name":"aggregate__3","type":"SQLAggregation","datasource":"actions_audit_filter","groupByParams":{"group_expression":"action_date, service_type"},"selectFunctions":[{"fieldName":"service_count","aggregationFunction":"SUM(action_count)"}],"isDefault":false},"data-writer__5":{"name":"data_writer__5","type":"IcebergWriter","iceberg_catalog":"dremio","warehouse_directory":"servicemetrics","datasource":"aggregate__3","mode":"merge","typeLabel":"Iceberg (Legacy)","unique_id":["action_date","service_type"],"isDefault":false}}},"blocks":[{"name":"actions_audit_reader","type":"SparkIcebergReader","options":{},"iceberg_catalog":"dremio","typeLabel":"Spark","isDefault":false,"table_name":"actionsaudit","region":"us-west-1","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"}},{"name":"actions_audit_mapper","type":"DataMapping","options":{},"fromDataReader":"actions_audit_reader","includeExistingColumns":false,"toSchema":[{"fieldName":"action_date","valueExpression":"DATE(action_date)"},{"fieldName":"service_type","valueExpression":"sub_category"},{"fieldName":"action_count","valueExpression":"action_count"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1756445643483","newFieldName":"action_date","mappingType":"expression","value":"DATE(action_date)"},{"id":"mapping-1756445668299","newFieldName":"service_type","mappingType":"sourceColumn","value":"sub_category"},{"id":"mapping-1756445681282","newFieldName":"action_count","mappingType":"sourceColumn","value":"action_count"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0},"isDefault":false},{"name":"actions_audit_filter","type":"Filter","options":{},"datasource":"actions_audit_mapper","condition":"action_date >=  COALESCE((SELECT MAX(DATE(action_date)) FROM dremio.servicemetrics), (SELECT MIN(action_date) FROM actions_audit_mapper_df))","isDefault":false},{"name":"aggregate__3","type":"SQLAggregation","options":{},"datasource":"actions_audit_filter","groupByParams":{"group_expression":"action_date, service_type"},"selectFunctions":[{"fieldName":"service_count","aggregationFunction":"SUM(action_count)"}],"isDefault":false},{"name":"data_writer__5","type":"IcebergWriter","options":{},"iceberg_catalog":"dremio","warehouse_directory":"servicemetrics","datasource":"aggregate__3","mode":"merge","typeLabel":"Iceberg (Legacy)","unique_id":["action_date","service_type"],"isDefault":false}]}}