04:45:16.623Z WARN a0906a61-20e0-4a1e-9d46-ffb608645125 (ServerContext): Failed to put OmicronZonesConfig {
        generation: Generation(
            10,
        ),
        zones: [
            OmicronZoneConfig {
                id: f6b15e85-1aab-480d-a0c0-4e238d670fd8 (service),
                filesystem_pool: Some(
                    ZpoolName {
                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                        kind: External,
                    },
                ),
                zone_type: Clickhouse {
                    address: [fd00:1122:3344:102::22]:8123,
                    dataset: OmicronZoneDataset {
                        pool_name: ZpoolName {
                            id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                            kind: External,
                        },
                    },
                },
            },
            OmicronZoneConfig {
                id: b9220730-36ff-4e0e-b447-1d31765d4119 (service),
                filesystem_pool: Some(
                    ZpoolName {
                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                        kind: External,
                    },
                ),
                zone_type: CockroachDb {
                    address: [fd00:1122:3344:102::23]:32221,
                    dataset: OmicronZoneDataset {
                        pool_name: ZpoolName {
                            id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                            kind: External,
                        },
                    },
                },
            },
            OmicronZoneConfig {
                id: 4bf0ef2b-d835-40cd-b658-9e63e0e8d05a (service),
                filesystem_pool: Some(
                    ZpoolName {
                        id: 335c91ea-5000-4a64-8680-5d35a0d3768b (zpool),
                        kind: External,
                    },
                ),
                zone_type: Crucible {
                    address: [fd00:1122:3344:102::e]:32345,
                    dataset: OmicronZoneDataset {
                        pool_name: ZpoolName {
                            id: 335c91ea-5000-4a64-8680-5d35a0d3768b (zpool),
                            kind: External,
                        },
                    },
                },
            },
            OmicronZoneConfig {
                id: df3a1197-9eac-4dbf-b88c-61027d38001d (service),
                filesystem_pool: Some(
                    ZpoolName {
                        id: 93d4d6e8-2d21-472e-9f60-3692627fd7c0 (zpool),
                        kind: External,
                    },
                ),
                zone_type: Crucible {
                    address: [fd00:1122:3344:102::9]:32345,
                    dataset: OmicronZoneDataset {
                        pool_name: ZpoolName {
                            id: 93d4d6e8-2d21-472e-9f60-3692627fd7c0 (zpool),
                            kind: External,
                        },
                    },
                },
            },
            OmicronZoneConfig {
                id: f9303fc1-46df-40db-b6b2-e1bb578c0c2b (service),
                filesystem_pool: Some(
                    ZpoolName {
                        id: 3651bfec-785b-4cd9-b5f9-249400a43635 (zpool),
                        kind: External,
                    },
                ),
                zone_type: Crucible {
                    address: [fd00:1122:3344:102::b]:32345,
                    dataset: OmicronZoneDataset {
                        pool_name: ZpoolName {
                            id: 3651bfec-785b-4cd9-b5f9-249400a43635 (zpool),
                            kind: External,
                        },
                    },
                },
            },
            OmicronZoneConfig {
                id: fca191f8-f0f3-43b3-910c-88ae415839f5 (service),
                filesystem_pool: Some(
                    ZpoolName {
                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                        kind: External,
                    },
                ),
                zone_type: Crucible {
                    address: [fd00:1122:3344:102::c]:32345,
                    dataset: OmicronZoneDataset {
                        pool_name: ZpoolName {
                            id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                            kind: External,
                        },
                    },
                },
            },
            OmicronZoneConfig {
                id: 609359ed-5ec1-4313-9d8c-b3314065c0da (service),
                filesystem_pool: Some(
                    ZpoolName {
                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                        kind: External,
                    },
                ),
                zone_type: InternalDns {
                    dataset: OmicronZoneDataset {
                        pool_name: ZpoolName {
                            id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                            kind: External,
                        },
                    },
                    http_address: [fd00:1122:3344:2::1]:5353,
                    dns_address: [fd00:1122:3344:2::1]:53,
                    gz_address: fd00:1122:3344:2::2,
                    gz_address_index: 0,
                },
            },
            OmicronZoneConfig {
                id: d715c8e9-75cb-456a-a8dc-4264bd636389 (service),
                filesystem_pool: Some(
                    ZpoolName {
                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                        kind: External,
                    },
                ),
                zone_type: InternalNtp {
                    address: [fd00:1122:3344:102::24]:123,
                },
            },
            OmicronZoneConfig {
                id: a0906a61-20e0-4a1e-9d46-ffb608645125 (service),
                filesystem_pool: Some(
                    ZpoolName {
                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),
                        kind: External,
                    },
                ),
                zone_type: Nexus {
                    internal_address: [fd00:1122:3344:102::21]:12221,
                    external_ip: 172.20.29.2,
                    nic: NetworkInterface {
                        id: 15c1a9ff-8078-4641-bac8-1f81b63a10f7,
                        kind: Service {
                            id: a0906a61-20e0-4a1e-9d46-ffb608645125,
                        },
                        name: Name(
                            "nexus-a0906a61-20e0-4a1e-9d46-ffb608645125",
                        ),
                        ip: 172.30.2.5,
                        mac: MacAddr(
                            MacAddr6(
                                [
                                    168,
                                    64,
                                    37,
                                    255,
                                    128,
                                    0,
                                ],
                            ),
                        ),
                        subnet: V4(
                            Ipv4Net {
                                addr: 172.30.2.0,
                                width: 24,
                            },
                        ),
                        vni: Vni(
                            100,
                        ),
                        primary: true,
                        slot: 0,
                        transit_ips: [],
                    },
                    external_tls: true,
                    external_dns_servers: [
                        1.1.1.1,
                        9.9.9.9,
                    ],
                },
            },
        ],
    } to sled d026352e-077d-4f6e-a31c-4f0b18b6d1ac: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "236145b0-67a8-460e-9057-17d7e3912099", "content-length": "124", "date": "Sat, 04 Jan 2025 04:45:16 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "236145b0-67a8-460e-9057-17d7e3912099" }
    background_task = blueprint_executor
    comment = sled d026352e-077d-4f6e-a31c-4f0b18b6d1ac: expunged 2 zones because: zone using expunged disk, sled d026352e-077d-4f6e-a31c-4f0b18b6d1ac: added 0 disks, updated 0, removed 0 disks, sled d026352e-077d-4f6e-a31c-4f0b18b6d1ac: added zone: boundary_ntp
    file = nexus/reconfigurator/execution/src/omicron_zones.rs:80
04:45:17.115Z DEBG a0906a61-20e0-4a1e-9d46-ffb608645125 (ServerContext): activation complete
    background_task = blueprint_executor
    elapsed = 10.100182228s
    iteration = 477
    status = {"enabled":true,"event_report":{"Ok":{"last_seen":5,"progress_events":[],"root_execution_id":"c390ffeb-18f7-4164-a81c-dce590a9cfdf","step_events":[{"data":{"components":[{"component":"ExternalNetworking","total_component_steps":1},{"component":"SledList","total_component_steps":1},{"component":"PhysicalDisks","total_component_steps":2},{"component":"Datasets","total_component_steps":1},{"component":"OmicronZones","total_component_steps":4},{"component":"FirewallRules","total_component_steps":1},{"component":"DatasetRecords","total_component_steps":1},{"component":"Dns","total_component_steps":1},{"component":"Clickhouse","total_component_steps":2},{"component":"Cockroach","total_component_steps":2}],"first_step":{"info":{"component":"ExternalNetworking","component_index":0,"description":"Ensure external networking resources","id":"Ensure","index":0,"total_component_steps":1},"metadata":null},"kind":"execution_started","steps":[{"component":"ExternalNetworking","component_index":0,"description":"Ensure external networking resources","id":"Ensure","index":0,"total_component_steps":1},{"component":"SledList","component_index":0,"description":"Fetch sled list","id":"Fetch","index":1,"total_component_steps":1},{"component":"PhysicalDisks","component_index":0,"description":"Deploy physical disks","id":"Ensure","index":2,"total_component_steps":2},{"component":"Datasets","component_index":0,"description":"Deploy datasets","id":"Ensure","index":3,"total_component_steps":1},{"component":"OmicronZones","component_index":0,"description":"Deploy Omicron zones","id":"Ensure","index":4,"total_component_steps":4},{"component":"FirewallRules","component_index":0,"description":"Plumb service firewall rules","id":"Ensure","index":5,"total_component_steps":1},{"component":"DatasetRecords","component_index":0,"description":"Ensure dataset records","id":"Ensure","index":6,"total_component_steps":1},{"component":"Dns","component_index":0,"description":"Deploy DNS records","id":"Ensure","index":7,"total_component_steps":1},{"component":"OmicronZones","component_index":1,"description":"Cleanup expunged zones","id":"Remove","index":8,"total_component_steps":4},{"component":"OmicronZones","component_index":2,"description":"Decommission sleds","id":"Remove","index":9,"total_component_steps":4},{"component":"PhysicalDisks","component_index":1,"description":"Decommission expunged disks","id":"Remove","index":10,"total_component_steps":2},{"component":"Clickhouse","component_index":0,"description":"Deploy clickhouse cluster nodes","id":"Ensure","index":11,"total_component_steps":2},{"component":"Clickhouse","component_index":1,"description":"Deploy single-node clickhouse cluster","id":"Ensure","index":12,"total_component_steps":2},{"component":"OmicronZones","component_index":3,"description":"Reassign sagas","id":"Ensure","index":13,"total_component_steps":4},{"component":"Cockroach","component_index":0,"description":"Ensure CockroachDB settings","id":"Ensure","index":14,"total_component_steps":2},{"component":"Cockroach","component_index":1,"description":"Finalize and check for errors","id":"Finalize","index":15,"total_component_steps":2}]},"event_index":0,"execution_id":"c390ffeb-18f7-4164-a81c-dce590a9cfdf","spec":"ReconfiguratorExecutionSpec","total_elapsed":{"nanos":2124,"secs":0}},{"data":{"attempt":1,"attempt_elapsed":{"nanos":67947938,"secs":0},"kind":"step_completed","next_step":{"info":{"component":"SledList","component_index":0,"description":"Fetch sled list","id":"Fetch","index":1,"total_component_steps":1},"metadata":null},"outcome":{"kind":"success","message":null,"metadata":null},"step":{"info":{"component":"ExternalNetworking","component_index":0,"description":"Ensure external networking resources","id":"Ensure","index":0,"total_component_steps":1},"metadata":null},"step_elapsed":{"nanos":67947868,"secs":0}},"event_index":1,"execution_id":"c390ffeb-18f7-4164-a81c-dce590a9cfdf","spec":"ReconfiguratorExecutionSpec","total_elapsed":{"nanos":67954790,"secs":0}},{"data":{"attempt":1,"attempt_elapsed":{"nanos":74388604,"secs":0},"kind":"step_completed","next_step":{"info":{"component":"PhysicalDisks","component_index":0,"description":"Deploy physical disks","id":"Ensure","index":2,"total_component_steps":2},"metadata":null},"outcome":{"kind":"success","message":null,"metadata":null},"step":{"info":{"component":"SledList","component_index":0,"description":"Fetch sled list","id":"Fetch","index":1,"total_component_steps":1},"metadata":null},"step_elapsed":{"nanos":74388524,"secs":0}},"event_index":2,"execution_id":"c390ffeb-18f7-4164-a81c-dce590a9cfdf","spec":"ReconfiguratorExecutionSpec","total_elapsed":{"nanos":142364825,"secs":0}},{"data":{"attempt":1,"attempt_elapsed":{"nanos":626268430,"secs":0},"kind":"step_completed","next_step":{"info":{"component":"Datasets","component_index":0,"description":"Deploy datasets","id":"Ensure","index":3,"total_component_steps":1},"metadata":null},"outcome":{"kind":"success","message":null,"metadata":null},"step":{"info":{"component":"PhysicalDisks","component_index":0,"description":"Deploy physical disks","id":"Ensure","index":2,"total_component_steps":2},"metadata":null},"step_elapsed":{"nanos":626268370,"secs":0}},"event_index":3,"execution_id":"c390ffeb-18f7-4164-a81c-dce590a9cfdf","spec":"ReconfiguratorExecutionSpec","total_elapsed":{"nanos":768655647,"secs":0}},{"data":{"attempt":1,"attempt_elapsed":{"nanos":842335210,"secs":1},"kind":"step_completed","next_step":{"info":{"component":"OmicronZones","component_index":0,"description":"Deploy Omicron zones","id":"Ensure","index":4,"total_component_steps":4},"metadata":null},"outcome":{"kind":"success","message":null,"metadata":null},"step":{"info":{"component":"Datasets","component_index":0,"description":"Deploy datasets","id":"Ensure","index":3,"total_component_steps":1},"metadata":null},"step_elapsed":{"nanos":842335169,"secs":1}},"event_index":4,"execution_id":"c390ffeb-18f7-4164-a81c-dce590a9cfdf","spec":"ReconfiguratorExecutionSpec","total_elapsed":{"nanos":611005735,"secs":2}},{"data":{"attempt_elapsed":{"nanos":488932771,"secs":7},"causes":["Error Response: status: 500 Internal Server Error; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"236145b0-67a8-460e-9057-17d7e3912099\\", \\"content-length\\": \\"124\\", \\"date\\": \\"Sat, 04 Jan 2025 04:45:16 GMT\\"}; value: Error { error_code: Some(\\"Internal\\"), message: \\"Internal Server Error\\", request_id: \\"236145b0-67a8-460e-9057-17d7e3912099\\" }"],"failed_step":{"info":{"component":"OmicronZones","component_index":0,"description":"Deploy Omicron zones","id":"Ensure","index":4,"total_component_steps":4},"metadata":null},"kind":"execution_failed","message":"Failed to put OmicronZonesConfig {\\n    generation: Generation(\\n        10,\\n    ),\\n    zones: [\\n        OmicronZoneConfig {\\n            id: f6b15e85-1aab-480d-a0c0-4e238d670fd8 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Clickhouse {\\n                address: [fd00:1122:3344:102::22]:8123,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: b9220730-36ff-4e0e-b447-1d31765d4119 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: CockroachDb {\\n                address: [fd00:1122:3344:102::23]:32221,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: 4bf0ef2b-d835-40cd-b658-9e63e0e8d05a (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 335c91ea-5000-4a64-8680-5d35a0d3768b (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Crucible {\\n                address: [fd00:1122:3344:102::e]:32345,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 335c91ea-5000-4a64-8680-5d35a0d3768b (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: df3a1197-9eac-4dbf-b88c-61027d38001d (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 93d4d6e8-2d21-472e-9f60-3692627fd7c0 (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Crucible {\\n                address: [fd00:1122:3344:102::9]:32345,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 93d4d6e8-2d21-472e-9f60-3692627fd7c0 (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: f9303fc1-46df-40db-b6b2-e1bb578c0c2b (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 3651bfec-785b-4cd9-b5f9-249400a43635 (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Crucible {\\n                address: [fd00:1122:3344:102::b]:32345,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 3651bfec-785b-4cd9-b5f9-249400a43635 (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: fca191f8-f0f3-43b3-910c-88ae415839f5 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Crucible {\\n                address: [fd00:1122:3344:102::c]:32345,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: 609359ed-5ec1-4313-9d8c-b3314065c0da (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: InternalDns {\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                        kind: External,\\n                    },\\n                },\\n                http_address: [fd00:1122:3344:2::1]:5353,\\n                dns_address: [fd00:1122:3344:2::1]:53,\\n                gz_address: fd00:1122:3344:2::2,\\n                gz_address_index: 0,\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: d715c8e9-75cb-456a-a8dc-4264bd636389 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: InternalNtp {\\n                address: [fd00:1122:3344:102::24]:123,\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: a0906a61-20e0-4a1e-9d46-ffb608645125 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Nexus {\\n                internal_address: [fd00:1122:3344:102::21]:12221,\\n                external_ip: 172.20.29.2,\\n                nic: NetworkInterface {\\n                    id: 15c1a9ff-8078-4641-bac8-1f81b63a10f7,\\n                    kind: Service {\\n                        id: a0906a61-20e0-4a1e-9d46-ffb608645125,\\n                    },\\n                    name: Name(\\n                        \\"nexus-a0906a61-20e0-4a1e-9d46-ffb608645125\\",\\n                    ),\\n                    ip: 172.30.2.5,\\n                    mac: MacAddr(\\n                        MacAddr6(\\n                            [\\n                                168,\\n                                64,\\n                                37,\\n                                255,\\n                                128,\\n                                0,\\n                            ],\\n                        ),\\n                    ),\\n                    subnet: V4(\\n                        Ipv4Net {\\n                            addr: 172.30.2.0,\\n                            width: 24,\\n                        },\\n                    ),\\n                    vni: Vni(\\n                        100,\\n                    ),\\n                    primary: true,\\n                    slot: 0,\\n                    transit_ips: [],\\n                },\\n                external_tls: true,\\n                external_dns_servers: [\\n                    1.1.1.1,\\n                    9.9.9.9,\\n                ],\\n            },\\n        },\\n    ],\\n} to sled d026352e-077d-4f6e-a31c-4f0b18b6d1ac","step_elapsed":{"nanos":488932711,"secs":7},"total_attempts":1},"event_index":5,"execution_id":"c390ffeb-18f7-4164-a81c-dce590a9cfdf","spec":"ReconfiguratorExecutionSpec","total_elapsed":{"nanos":99953164,"secs":10}}]}},"execution_error":{"causes":["Failed to put OmicronZonesConfig {\\n    generation: Generation(\\n        10,\\n    ),\\n    zones: [\\n        OmicronZoneConfig {\\n            id: f6b15e85-1aab-480d-a0c0-4e238d670fd8 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Clickhouse {\\n                address: [fd00:1122:3344:102::22]:8123,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: b9220730-36ff-4e0e-b447-1d31765d4119 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: CockroachDb {\\n                address: [fd00:1122:3344:102::23]:32221,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: 4bf0ef2b-d835-40cd-b658-9e63e0e8d05a (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 335c91ea-5000-4a64-8680-5d35a0d3768b (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Crucible {\\n                address: [fd00:1122:3344:102::e]:32345,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 335c91ea-5000-4a64-8680-5d35a0d3768b (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: df3a1197-9eac-4dbf-b88c-61027d38001d (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 93d4d6e8-2d21-472e-9f60-3692627fd7c0 (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Crucible {\\n                address: [fd00:1122:3344:102::9]:32345,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 93d4d6e8-2d21-472e-9f60-3692627fd7c0 (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: f9303fc1-46df-40db-b6b2-e1bb578c0c2b (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 3651bfec-785b-4cd9-b5f9-249400a43635 (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Crucible {\\n                address: [fd00:1122:3344:102::b]:32345,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 3651bfec-785b-4cd9-b5f9-249400a43635 (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: fca191f8-f0f3-43b3-910c-88ae415839f5 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Crucible {\\n                address: [fd00:1122:3344:102::c]:32345,\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                        kind: External,\\n                    },\\n                },\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: 609359ed-5ec1-4313-9d8c-b3314065c0da (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: InternalDns {\\n                dataset: OmicronZoneDataset {\\n                    pool_name: ZpoolName {\\n                        id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                        kind: External,\\n                    },\\n                },\\n                http_address: [fd00:1122:3344:2::1]:5353,\\n                dns_address: [fd00:1122:3344:2::1]:53,\\n                gz_address: fd00:1122:3344:2::2,\\n                gz_address_index: 0,\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: d715c8e9-75cb-456a-a8dc-4264bd636389 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: InternalNtp {\\n                address: [fd00:1122:3344:102::24]:123,\\n            },\\n        },\\n        OmicronZoneConfig {\\n            id: a0906a61-20e0-4a1e-9d46-ffb608645125 (service),\\n            filesystem_pool: Some(\\n                ZpoolName {\\n                    id: 083c7773-26c6-43be-a091-96355e10e30f (zpool),\\n                    kind: External,\\n                },\\n            ),\\n            zone_type: Nexus {\\n                internal_address: [fd00:1122:3344:102::21]:12221,\\n                external_ip: 172.20.29.2,\\n                nic: NetworkInterface {\\n                    id: 15c1a9ff-8078-4641-bac8-1f81b63a10f7,\\n                    kind: Service {\\n                        id: a0906a61-20e0-4a1e-9d46-ffb608645125,\\n                    },\\n                    name: Name(\\n                        \\"nexus-a0906a61-20e0-4a1e-9d46-ffb608645125\\",\\n                    ),\\n                    ip: 172.30.2.5,\\n                    mac: MacAddr(\\n                        MacAddr6(\\n                            [\\n                                168,\\n                                64,\\n                                37,\\n                                255,\\n                                128,\\n                                0,\\n                            ],\\n                        ),\\n                    ),\\n                    subnet: V4(\\n                        Ipv4Net {\\n                            addr: 172.30.2.0,\\n                            width: 24,\\n                        },\\n                    ),\\n                    vni: Vni(\\n                        100,\\n                    ),\\n                    primary: true,\\n                    slot: 0,\\n                    transit_ips: [],\\n                },\\n                external_tls: true,\\n                external_dns_servers: [\\n                    1.1.1.1,\\n                    9.9.9.9,\\n                ],\\n            },\\n        },\\n    ],\\n} to sled d026352e-077d-4f6e-a31c-4f0b18b6d1ac","Error Response: status: 500 Internal Server Error; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"236145b0-67a8-460e-9057-17d7e3912099\\", \\"content-length\\": \\"124\\", \\"date\\": \\"Sat, 04 Jan 2025 04:45:16 GMT\\"}; value: Error { error_code: Some(\\"Internal\\"), message: \\"Internal Server Error\\", request_id: \\"236145b0-67a8-460e-9057-17d7e3912099\\" }"],"message":"step failed: Deploy Omicron zones"},"target_id":"c05543fc-cce7-485e-b763-7a30d1d8e070"}
04:45:22.542Z ERRO a0906a61-20e0-4a1e-9d46-ffb608645125 (ServerContext): failed to ensure service zone nat entry
    background_task = service_zone_nat_tracker
    entry = Ipv4NatValues { external_address: Ipv4Net(Ipv4Net { addr: 172.20.29.5, width: 32 }), first_port: SqlU16(16384), last_port: SqlU16(32767), sled_address: Ipv6Net(Ipv6Net { addr: fd00:1122:3344:103::1, width: 128 }), vni: Vni(Vni(100)), mac: MacAddr(MacAddr(MacAddr6([168, 64, 37, 255, 128, 1]))) }
    error = InternalError { internal_message: "unexpected database error: duplicate key value violates unique constraint \\"overlapping_ipv4_nat_entry\\"" }
    file = nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs:166

root@[fd00:1122:3344:102::23]:32221/omicron> select * from external_ip where time_deleted is NULL and is_service;
                   id                  |                       name                        | description  |         time_created          |         time_modified         | time_deleted |              ip_pool_id              |           ip_pool_range_id           | is_service |              parent_id               |   kind   |     ip      | first_port | last_port | project_id |  state   | is_probe
---------------------------------------+---------------------------------------------------+--------------+-------------------------------+-------------------------------+--------------+--------------------------------------+--------------------------------------+------------+--------------------------------------+----------+-------------+------------+-----------+------------+----------+-----------
  17c723ea-2e69-4afa-b370-a781d209faaf | nexus-2772ac01-b24b-4a5e-b116-b1ec779dc897        | nexus        | 2024-12-18 21:14:00.805113+00 | 2024-12-18 21:14:00.805113+00 | NULL         | 19515548-8f12-4a56-b708-ba137b505a82 | 65aead27-1e2d-4ad8-b35f-abd544086a8f |    true    | 2772ac01-b24b-4a5e-b116-b1ec779dc897 | floating | 172.20.29.3 |          0 |     65535 | NULL       | attached |  false
  26dbb4d4-4b04-4a3c-93bb-d17f06b4d290 | external-dns-94b865bc-b332-4664-a4a5-20a6386938d5 | external_dns | 2024-12-18 21:14:00.665571+00 | 2024-12-18 21:14:00.665571+00 | NULL         | 19515548-8f12-4a56-b708-ba137b505a82 | 65aead27-1e2d-4ad8-b35f-abd544086a8f |    true    | 94b865bc-b332-4664-a4a5-20a6386938d5 | floating | 172.20.29.1 |          0 |     65535 | NULL       | attached |  false
  4a97f234-b4ca-4f50-aeec-e12fc83aaabd | nexus-17d82559-8c9d-4d1b-bb53-2b45f43a1a30        | nexus        | 2024-12-18 21:14:00.719294+00 | 2024-12-18 21:14:00.719294+00 | NULL         | 19515548-8f12-4a56-b708-ba137b505a82 | 65aead27-1e2d-4ad8-b35f-abd544086a8f |    true    | 17d82559-8c9d-4d1b-bb53-2b45f43a1a30 | floating | 172.20.29.4 |          0 |     65535 | NULL       | attached |  false
  4c01d6cf-50ce-4fd1-9d9b-a238a4f5921c | NULL                                              | NULL         | 2024-12-18 21:14:00.692632+00 | 2024-12-18 21:14:00.692632+00 | NULL         | 19515548-8f12-4a56-b708-ba137b505a82 | 65aead27-1e2d-4ad8-b35f-abd544086a8f |    true    | fe6953cb-a4ca-43ab-bd6b-fc5aaf96321a | snat     | 172.20.29.5 |          0 |     16383 | NULL       | attached |  false
  c1b7e64d-900b-43c9-8175-c8486d4c7d56 | nexus-a0906a61-20e0-4a1e-9d46-ffb608645125        | nexus        | 2025-01-03 20:51:03.223499+00 | 2025-01-03 20:51:03.223499+00 | NULL         | 19515548-8f12-4a56-b708-ba137b505a82 | 65aead27-1e2d-4ad8-b35f-abd544086a8f |    true    | a0906a61-20e0-4a1e-9d46-ffb608645125 | floating | 172.20.29.2 |          0 |     65535 | NULL       | attached |  false
  f24629f9-a632-47c7-b0ce-df255836aac0 | NULL                                              | NULL         | 2025-01-04 04:44:22.191357+00 | 2025-01-04 04:44:22.191357+00 | NULL         | 19515548-8f12-4a56-b708-ba137b505a82 | 65aead27-1e2d-4ad8-b35f-abd544086a8f |    true    | 55f77901-e614-4595-9661-196f3e264f90 | snat     | 172.20.29.5 |      16384 |     32767 | NULL       | attached |  false
(6 rows)