{"version":"v1alpha","kind":"VisualBuilder","metadata":{"name":"service_request_metrics","description":"Visual builder workflow for service request metrics","runtime":"spark"},"spec":{"ui":{"edges":[{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-reader__0","target":"data-mapper__1","id":"xy-edge__data-reader__0-data-mapper__1"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"data-mapper__1","target":"filter__2","id":"xy-edge__data-mapper__1-filter__2"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"filter__2","target":"aggregate__3","id":"xy-edge__filter__2-aggregate__3"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"aggregate__3","target":"code-transform__0","id":"xy-edge__aggregate__3-code-transform__0"},{"style":{"stroke":"var(--color-primary-lighter)","strokeWidth":1},"source":"code-transform__0","target":"data-writer__4","id":"xy-edge__code-transform__0-data-writer__4"}],"nodes":[{"id":"data-reader__0","type":"workflowNode","position":{"x":589,"y":131},"data":{"nodeType":"data-reader","id":"data-reader__0"},"measured":{"width":240,"height":144},"selected":false,"dragging":false,"style":{}},{"id":"data-mapper__1","type":"workflowNode","position":{"x":1044.5,"y":131.5},"data":{"nodeType":"data-mapper","id":"data-mapper__1"},"measured":{"width":240,"height":144},"selected":false,"dragging":false,"style":{}},{"id":"filter__2","type":"workflowNode","position":{"x":1521.25,"y":125.75},"data":{"nodeType":"filter","id":"filter__2"},"measured":{"width":240,"height":168},"selected":false,"dragging":false,"style":{}},{"id":"aggregate__3","type":"workflowNode","position":{"x":2003.25,"y":121.75},"data":{"nodeType":"aggregate","id":"aggregate__3"},"measured":{"width":240,"height":172},"selected":false,"dragging":false,"style":{}},{"id":"data-writer__4","type":"workflowNode","position":{"x":2819.27194734189,"y":74.1320621195083},"data":{"nodeType":"data-writer","id":"data-writer__4"},"measured":{"width":240,"height":200},"selected":false,"dragging":false,"style":{}},{"id":"code-transform__0","type":"workflowNode","position":{"x":2411.932611772461,"y":81.33940728526409},"data":{"nodeType":"code-transform","id":"code-transform__0"},"measured":{"width":240,"height":144},"selected":true,"dragging":false,"style":{}}],"nodesData":{"data-reader__0":{"typeLabel":"Spark","isDefault":false,"name":"ActionsAuditData","type":"SparkReader","format":"iceberg","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"iceberg_catalog":"dremio","region":"us-west-1","table_name":"actionsaudit","connectedComponents":[],"columns":[]},"data-mapper__1":{"name":"data_mapper__1","type":"DataMapping","fromDataReader":"ActionsAuditData","includeExistingColumns":false,"toSchema":[{"fieldName":"action_date","valueExpression":"DATE(action_date)"},{"fieldName":"service_type","valueExpression":"sub_category"},{"fieldName":"action_count","valueExpression":"action_count"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1753983206926","newFieldName":"action_date","mappingType":"expression","value":"DATE(action_date)"},{"id":"mapping-1753983229372","newFieldName":"service_type","mappingType":"sourceColumn","value":"sub_category"},{"id":"mapping-1753983263800","newFieldName":"action_count","mappingType":"sourceColumn","value":"action_count"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0,"selectedSourceSystem":"","selectedTargetSystem":"","selectedSourceLayout":"","selectedTargetLayout":"","selectedTargetLayoutFile":"","manualTargetMappings":[],"confirmedGlossaryMappings":[],"confirmedManualTargetMappings":[],"confirmedSSEMappings":[],"glossaryAssistedMappings":[],"additionalFieldMappings":[]},"isDefault":false,"datasource":"ActionsAuditData","connectedComponents":["ActionsAuditData"]},"filter__2":{"name":"LatestServiceRequests","type":"Filter","datasource":"data_mapper__1","condition":"action_date >=  COALESCE((SELECT MAX(DATE(action_date)) FROM dremio.servicemetrics), (SELECT MIN(action_date) FROM data_mapper__1_df))","isDefault":false},"aggregate__3":{"name":"aggregate__3","type":"SQLAggregation","datasource":"LatestServiceRequests","groupByParams":{"group_expression":"action_date, service_type"},"selectFunctions":[{"fieldName":"service_count","aggregationFunction":"sum('action_count')"}],"isDefault":false,"materialization_strategy":"NONE","fail_on_error":true,"connectedComponents":["LatestServiceRequests"]},"data-writer__4":{"name":"ServiceRequestMetricsWriter","type":"IcebergWriter","iceberg_catalog":"dremio","warehouse_directory":"servicemetrics","datasource":"CheckpointOutput","mode":"merge","typeLabel":"Iceberg (Legacy)","unique_id":["action_date","service_type"],"isDefault":false,"connectedComponents":["CheckpointOutput"]},"code-transform__0":{"name":"CheckpointOutput","type":"CodeTransform","language":"python","datasource":"aggregate__3","code":"CheckpointOutput_df = {{datasource}}_df.localCheckpoint()\n{{datasource}}_df.persist()\nCheckpointOutput_df.createOrReplaceTempView(\"CheckpointOutput_df\")","materialization_strategy":"NONE","fail_on_error":true,"isDefault":false,"connectedComponents":["aggregate__3"]}}},"blocks":[{"name":"ActionsAuditData","type":"SparkReader","options":{},"typeLabel":"Spark","isDefault":false,"format":"iceberg","credentials":{"accessKey":"S3_ACCESS_KEY","secretKey":"S3_SECRET_KEY"},"iceberg_catalog":"dremio","region":"us-west-1","table_name":"actionsaudit","connectedComponents":[],"columns":[]},{"name":"data_mapper__1","type":"DataMapping","options":{},"fromDataReader":"ActionsAuditData","includeExistingColumns":false,"toSchema":[{"fieldName":"action_date","valueExpression":"DATE(action_date)"},{"fieldName":"service_type","valueExpression":"sub_category"},{"fieldName":"action_count","valueExpression":"action_count"}],"additionalData":{"isGlossaryAssisted":false,"selectedSourceGlossary":"","selectedTargetGlossary":"","manualMappings":[{"id":"mapping-1753983206926","newFieldName":"action_date","mappingType":"expression","value":"DATE(action_date)"},{"id":"mapping-1753983229372","newFieldName":"service_type","mappingType":"sourceColumn","value":"sub_category"},{"id":"mapping-1753983263800","newFieldName":"action_count","mappingType":"sourceColumn","value":"action_count"}],"confirmedMapping":[],"mappingBatchKeysAfterConfirm":[],"after":"","totalSourceTerms":0,"selectedSourceSystem":"","selectedTargetSystem":"","selectedSourceLayout":"","selectedTargetLayout":"","selectedTargetLayoutFile":"","manualTargetMappings":[],"confirmedGlossaryMappings":[],"confirmedManualTargetMappings":[],"confirmedSSEMappings":[],"glossaryAssistedMappings":[],"additionalFieldMappings":[]},"isDefault":false,"datasource":"ActionsAuditData","connectedComponents":["ActionsAuditData"]},{"name":"LatestServiceRequests","type":"Filter","options":{},"datasource":"data_mapper__1","condition":"action_date >=  COALESCE((SELECT MAX(DATE(action_date)) FROM dremio.servicemetrics), (SELECT MIN(action_date) FROM data_mapper__1_df))","isDefault":false},{"name":"aggregate__3","type":"SQLAggregation","options":{},"datasource":"LatestServiceRequests","groupByParams":{"group_expression":"action_date, service_type"},"selectFunctions":[{"fieldName":"service_count","aggregationFunction":"sum('action_count')"}],"isDefault":false,"materialization_strategy":"NONE","fail_on_error":true,"connectedComponents":["LatestServiceRequests"]},{"name":"ServiceRequestMetricsWriter","type":"IcebergWriter","options":{},"iceberg_catalog":"dremio","warehouse_directory":"servicemetrics","datasource":"CheckpointOutput","mode":"merge","typeLabel":"Iceberg (Legacy)","unique_id":["action_date","service_type"],"isDefault":false,"connectedComponents":["CheckpointOutput"]},{"name":"CheckpointOutput","type":"CodeTransform","options":{},"language":"python","datasource":"aggregate__3","code":"CheckpointOutput_df = {{datasource}}_df.localCheckpoint()\n{{datasource}}_df.persist()\nCheckpointOutput_df.createOrReplaceTempView(\"CheckpointOutput_df\")","materialization_strategy":"NONE","fail_on_error":true,"isDefault":false,"connectedComponents":["aggregate__3"]}]}}