{"version":"v1alpha","kind":"VisualBuilder","metadata":{"name":"service_request_metrics","description":"Visual builder workflow for service request metrics","runtime":"spark"},"spec":{"ui":{"edges":[{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__0","target":"data-mapper__1","id":"xy-edge__data-reader__0-data-mapper__1"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__1","target":"filter__2","id":"xy-edge__data-mapper__1-filter__2"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__2","target":"aggregate__3","id":"xy-edge__filter__2-aggregate__3"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__3","target":"code-transform__0","id":"xy-edge__aggregate__3-code-transform__0"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"code-transform__0","target":"data-writer__4","id":"xy-edge__code-transform__0-data-writer__4"}],"nodes":[{"id":"data-reader__0","type":"workflowNode","position":{"x":604.0778824915546,"y":95.19002908255774},"data":{"nodeType":"data-reader","id":"data-reader__0"},"measured":{"width":240,"height":105},"selected":false,"dragging":false,"style":{}},{"id":"data-mapper__1","type":"workflowNode","position":{"x":1048.2694706228885,"y":97.57476439400207},"data":{"nodeType":"data-mapper","id":"data-mapper__1"},"measured":{"width":240,"height":105},"selected":false,"dragging":false,"style":{}},{"id":"filter__2","type":"workflowNode","position":{"x":1530.6736765572216,"y":63.55373472233714},"data":{"nodeType":"filter","id":"filter__2"},"measured":{"width":240,"height":149},"selected":false,"dragging":false,"style":{}},{"id":"aggregate__3","type":"workflowNode","position":{"x":1991.941588131334,"y":59.55373472233714},"data":{"nodeType":"aggregate","id":"aggregate__3"},"measured":{"width":240,"height":105},"selected":false,"dragging":false,"style":{}},{"id":"data-writer__4","type":"workflowNode","position":{"x":2666.9319322361207,"y":51.56742156854904},"data":{"nodeType":"data-writer","id":"data-writer__4"},"measured":{"width":240,"height":129},"selected":false,"dragging":false,"style":{}},{"id":"code-transform__0","type":"workflowNode","position":{"x":2332.304118525907,"y":61.39336266399562},"data":{"nodeType":"code-transform","id":"code-transform__0"},"measured":{"width":240,"height":105},"selected":false,"dragging":false,"style":{}}],"nodesData":{"data-reader__0":{"typeLabel":"Spark","isDefault":false,"connectedComponents":[],"columns":[],"name":"ActionsAuditData","type":"SparkReader","format":"iceberg","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"iceberg_catalog":"dremio","region":"us-west-1","table_name":"actionsaudit"},"data-mapper__1":{"name":"data_mapper__1","type":"DataMapping","fromDataReader":"ActionsAuditData","includeExistingColumns":false,"toSchema":[{"fieldName":"action_date","valueExpression":"DATE(action_date)"},{"fieldName":"service_type","valueExpression":"sub_category"},{"fieldName":"action_count","valueExpression":"action_count"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1753983206926","newFieldName":"action_date","mappingType":"expression","value":"DATE(action_date)"},{"id":"mapping-1753983229372","newFieldName":"service_type","mappingType":"sourceColumn","value":"sub_category"},{"id":"mapping-1753983263800","newFieldName":"action_count","mappingType":"sourceColumn","value":"action_count"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0,"selectedSourceSystem":"","selectedTargetSystem":"","selectedSourceLayout":"","selectedTargetLayout":"","selectedTargetLayoutFile":"","manualTargetMappings":[],"confirmedGlossaryMappings":[],"confirmedManualTargetMappings":[],"confirmedSSEMappings":[],"glossaryAssistedMappings":[],"additionalFieldMappings":[]},"isDefault":false,"datasource":"ActionsAuditData","connectedComponents":["ActionsAuditData"]},"filter__2":{"name":"LatestServiceRequests","type":"Filter","datasource":"data_mapper__1","condition":"action_date >=  COALESCE((SELECT MAX(DATE(action_date)) FROM dremio.servicemetrics), (SELECT MIN(action_date) FROM data_mapper__1_df))","isDefault":false},"aggregate__3":{"name":"aggregate__3","type":"SQLAggregation","datasource":"LatestServiceRequests","groupByParams":{"group_expression":"action_date, service_type"},"selectFunctions":[{"fieldName":"service_count","aggregationFunction":"sum('action_count')"}],"isDefault":false,"materialization_strategy":"NONE","fail_on_error":true,"connectedComponents":["LatestServiceRequests"]},"data-writer__4":{"name":"ServiceRequestMetricsWriter","type":"SparkWriter","format":"iceberg","mode":"append","datasource":"CheckpointOutput","typeLabel":"Spark","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"iceberg_catalog":"dremio","region":"us-west-1","table_name":"servicemetrics","isDefault":false,"connectedComponents":["CheckpointOutput"]},"code-transform__0":{"name":"CheckpointOutput","type":"CodeTransform","language":"python","datasource":"aggregate__3","code":"CheckpointOutput_df = {{datasource}}_df.localCheckpoint()\n# {{datasource}}_df.persist()\nCheckpointOutput_df.createOrReplaceTempView(\"CheckpointOutput_df\")","materialization_strategy":"NONE","fail_on_error":true,"isDefault":false,"connectedComponents":["aggregate__3"]}}},"blocks":[{"name":"ActionsAuditData","type":"SparkReader","options":{},"typeLabel":"Spark","isDefault":false,"connectedComponents":[],"columns":[],"format":"iceberg","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"iceberg_catalog":"dremio","region":"us-west-1","table_name":"actionsaudit"},{"name":"data_mapper__1","type":"DataMapping","options":{},"fromDataReader":"ActionsAuditData","includeExistingColumns":false,"toSchema":[{"fieldName":"action_date","valueExpression":"DATE(action_date)"},{"fieldName":"service_type","valueExpression":"sub_category"},{"fieldName":"action_count","valueExpression":"action_count"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1753983206926","newFieldName":"action_date","mappingType":"expression","value":"DATE(action_date)"},{"id":"mapping-1753983229372","newFieldName":"service_type","mappingType":"sourceColumn","value":"sub_category"},{"id":"mapping-1753983263800","newFieldName":"action_count","mappingType":"sourceColumn","value":"action_count"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0,"selectedSourceSystem":"","selectedTargetSystem":"","selectedSourceLayout":"","selectedTargetLayout":"","selectedTargetLayoutFile":"","manualTargetMappings":[],"confirmedGlossaryMappings":[],"confirmedManualTargetMappings":[],"confirmedSSEMappings":[],"glossaryAssistedMappings":[],"additionalFieldMappings":[]},"isDefault":false,"datasource":"ActionsAuditData","connectedComponents":["ActionsAuditData"]},{"name":"LatestServiceRequests","type":"Filter","options":{},"datasource":"data_mapper__1","condition":"action_date >=  COALESCE((SELECT MAX(DATE(action_date)) FROM dremio.servicemetrics), (SELECT MIN(action_date) FROM data_mapper__1_df))","isDefault":false},{"name":"aggregate__3","type":"SQLAggregation","options":{},"datasource":"LatestServiceRequests","groupByParams":{"group_expression":"action_date, service_type"},"selectFunctions":[{"fieldName":"service_count","aggregationFunction":"sum('action_count')"}],"isDefault":false,"materialization_strategy":"NONE","fail_on_error":true,"connectedComponents":["LatestServiceRequests"]},{"name":"ServiceRequestMetricsWriter","type":"SparkWriter","options":{},"format":"iceberg","mode":"append","datasource":"CheckpointOutput","typeLabel":"Spark","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"iceberg_catalog":"dremio","region":"us-west-1","table_name":"servicemetrics","isDefault":false,"connectedComponents":["CheckpointOutput"]},{"name":"CheckpointOutput","type":"CodeTransform","options":{},"language":"python","datasource":"aggregate__3","code":"CheckpointOutput_df = {{datasource}}_df.localCheckpoint()\n# {{datasource}}_df.persist()\nCheckpointOutput_df.createOrReplaceTempView(\"CheckpointOutput_df\")","materialization_strategy":"NONE","fail_on_error":true,"isDefault":false,"connectedComponents":["aggregate__3"]}]}}